def validate_system_accounts(snapshot): step('Verifying system accounts') found = [] for name, account in snapshot['accounts'].iteritems(): tick() if name in system_accounts: if account['privileged'] != system_accounts[name]['privileged']: fail() print "> %s account wrong privileged setting" % (name) return False # Verify resignement if name != "eosio.null" and name != "eosio.prods": actor = system_accounts[name]['actor'] permission = system_accounts[name]['permission'] if authority_controlled_by_one_actor(account['permissions']["owner"], actor, permission) != True or \ authority_controlled_by_one_actor(account['permissions']["owner"], actor, permission) != True: fail() print "> %s account NOT PROPERLY RESIGNED" % (name) return False found.append(name) not_found = set(system_accounts.keys()) - set(found) if len(not_found): fail() print "> missing system accounts %s" % (','.join(not_found)) return False success() return True
def retry(cmd): while True: with open(os.devnull, 'w') as devnull: p = Popen(cmd, stdout=PIPE, stderr=devnull) output, err = p.communicate("") if p.returncode: tick() else: break return output
def load_erc20_snapshot(args): step('Loading ETH snapshot') with open(args.csv_balance, 'rb') as csvfile: balances = {} for row in csv.reader(csvfile, delimiter=','): tick() row = [unicode(i) for i in row] row[CSV_EOS_BALANCE] = '{0} {1}'.format(row[CSV_EOS_BALANCE], args.core_symbol) balances[row[CSV_EOS_ACCOUNT]] = row success() return balances
def real_sell(): conn = sqlite3.connect("%s/%s" % (config.db_dir, config.db_file)) c = conn.cursor() for train in c.execute( "select tid,created_at, updated_at, buy, sell, t, n, increase, currency from trade, features where trade.digest = features.digest and updated_at == 0" ): tid, created_at, updated_at, buy, sell, t, n, increase, currency = train ticker = utils.tick(currency) tbuy, tsell = float(ticker["ticker"]["buy"]), float( ticker["ticker"]["sell"]) print "real", buy, tbuy, tbuy / buy * 100 - 100, increase, t, ( int(time.time() * 1000) - created_at) / 1000 if int(time.time() * 1000) > int( created_at ) + utils.str2sec(t) * n * 1000 or tbuy / buy * 100 - 100 > float( increase) or tbuy / buy * 100 - 100 < -1.0 * float(increase): #if True: r = trade(currency, "s", 1.0) if r != False: c.execute( "update trade set updated_at = ?, sell = ? where tid = ?", (r["trade_date"], r["price"], tid)) conn.commit() print "real sell %s @ %s, money %s" % (currency, r["price"], r["trade_money"]) conn.close()
def validate_no_code_in_accounts(snapshot, balances): step('Verifying user account code') with_code = 0 for account in balances: tick() acnt = snapshot['accounts'][account] if int('0x'+acnt['code_version'],16) != 0 or \ acnt['abi'] != "" or \ acnt['vm_type'] != 0 or \ acnt['vm_version'] != 0: with_code += 1 if with_code > 0: warning() print "> %d accounts with code set" % (with_code) else: success()
def validate_account_creation(snapshot, balances): found_users = {} print_count = 0 step('Verifying user account creation') for account in balances.keys(): tick() if account not in snapshot['accounts']: print_count = print_some(print_count, account) continue found_users[account] = balances[account] not_found = len(balances) - len(found_users) if not_found: print "> %d (out of %d) accounts were not found in the EOS snapshot" % ( not_found, len(balances)) else: success() return found_users
def train_sell(): conn = sqlite3.connect("%s/%s" % (config.db_dir, config.db_file)) c = conn.cursor() for train in c.execute( "select tid,created_at, updated_at, buy, sell, t, n, increase, currency from training, features where training.digest = features.digest and updated_at == 0" ): tid, created_at, updated_at, buy, sell, t, n, increase, currency = train ticker = utils.tick(currency) tbuy, tsell = float(ticker["ticker"]["buy"]), float( ticker["ticker"]["sell"]) print buy, tbuy, tbuy / buy * 100 - 100, increase, t, ( int(time.time() * 1000) - created_at) / 1000 if int(time.time() * 1000) > int( created_at ) + utils.str2sec(t) * n * 1000 or tbuy / buy * 100 - 100 > float( increase) or tbuy / buy * 100 - 100 < -1.0 * float(increase): c.execute( "update training set updated_at = ?, sell = ? where tid = ?", (int(time.time() * 1000), tbuy, tid)) conn.commit() print "sell %s @ %s" % (currency, tbuy) conn.close()
def train_buy(): conn = sqlite3.connect("%s/%s" % (config.db_dir, config.db_file)) c = conn.cursor() k = {} c.execute( "select digest, content, currency, t, size, n, increase, feature_size, point from features order by point" ) for f in c.fetchall(): digest, content, currency, t, size, n, increase, feature_size, point = f if "%s_%s_%s" % (currency, t, size) not in k: k["%s_%s_%s" % (currency, t, size)] = utils.kline( currency, t, "", size) print "Comparing to %s @ %s %s %s" % ( digest, datetime.datetime.fromtimestamp( point / 1000).strftime('%Y-%m-%d %H:%M:%S'), increase, t) s = kline_similarity(k["%s_%s_%s" % (currency, t, size)], json.loads(content), feature_size) v = float(json.loads(content)["v"]) if not math.isnan(s) and s <= v: created_at = int(time.time() * 1000) c.execute( "select * from training where digest = ? and created_at > ? and updated_at = 0", (digest, created_at - 1000 * n * utils.str2sec(t))) if len(c.fetchall()) > 0: continue print "Match" ticker = utils.tick(currency) buy, sell = float(ticker["ticker"]["buy"]), float( ticker["ticker"]["sell"]) print "buy %s @ %s" % (currency, sell) c.execute( "Insert into training (created_at, updated_at, digest, buy, sell) values (?, 0, ?, ?, 0.0)", (created_at, digest, sell)) conn.commit() else: #print "Not Match" pass conn.close()
def train_sell(): access_key = config.access_key access_secret = config.access_secret chbtc = api.chbtcApi(access_key, access_secret) account = chbtc.get_account_info() currency = "btc_cny" a = float( account["result"]["balance"][currency.split("_")[0].upper()]["amount"]) b = float(account["result"]["balance"][currency.split("_")[-1].upper()] ["amount"]) if a == 0.0: print "out of currency" return False conn = sqlite3.connect("%s/%s" % (config.db_dir, config.db_file)) c = conn.cursor() for train in c.execute( "select tid,created_at, updated_at, buy, sell, t, n, increase, currency from training, features where training.digest = features.digest and updated_at == 0" ): tid, created_at, updated_at, buy, sell, t, n, increase, currency = train ticker = utils.tick(currency) tbuy, tsell = float(ticker["ticker"]["buy"]), float( ticker["ticker"]["sell"]) print buy, tbuy, tbuy / buy * 100 - 100, increase, t, ( int(time.time() * 1000) - created_at) / 1000 if int(time.time() * 1000) > int( created_at ) + utils.str2sec(t) * n * 1000 or tbuy / buy * 100 - 100 > float( increase) or tbuy / buy * 100 - 100 < -1.0 * float(increase): c.execute( "update training set updated_at = ?, sell = ? where tid = ?", (int(time.time() * 1000), tbuy, tid)) conn.commit() print "sell %s @ %s" % (currency, tbuy) conn.close()
def order(currency, t="b", radio=1.0): access_key = config.access_key access_secret = config.access_secret chbtc = api.chbtcApi(access_key, access_secret) account = chbtc.get_account_info() a = float( account["result"]["balance"][currency.split("_")[0].upper()]["amount"]) b = float(account["result"]["balance"][currency.split("_")[-1].upper()] ["amount"]) tick = utils.tick(currency) sell = float(tick["ticker"]["sell"]) buy = float(tick["ticker"]["buy"]) if t == "b": t = 1 price = "%.3f" % (sell * 0.9999, ) #amount = "%.3f" % (b / sell * radio, ) amount = "%.3f" % (50.0 / sell * radio, ) else: t = 0 price = "%.3f" % (buy * 1.0001, ) #amount = "%.3f" % (a * radio, ) amount = "%.3f" % (0.01 * radio, ) if a < 0.01 and t == 0: amount = "%.3f" % (a, ) if b < 50.0 and t == 1: amount = "%.3f" % (b / (sell * 1.01), ) if float(amount) < 0.001: return None r = chbtc.order(price, amount, t, currency) return r
def load_eos_snapshot(args): step('Loading EOS snapshot') with open(args.snapshot) as file: snapshot = json.load(file) # Convert list to dictionary accounts = {} for acc in snapshot['accounts']: tick() accounts[acc['name']] = acc snapshot['accounts'] = accounts # Add permissions to account dict for perm in snapshot['permissions']: tick() if perm['id']['_id'] == 0: continue owner = perm['owner'] if owner not in snapshot['accounts']: raise ValidationException( "invalid snapshot: permission owner not found") if 'permissions' not in snapshot['accounts'][owner]: snapshot['accounts'][owner]['permissions'] = {} snapshot['accounts'][owner]['permissions'][perm['name']] = perm # Tables tables = {} for t in snapshot['tables']: tick() code = t['tid']['code'] scope = t['tid']['scope'] table = t['tid']['table'] if code not in tables: tables[code] = {} if scope not in tables[code]: tables[code][scope] = {} tables[code][scope][table] = t['rows'] snapshot['tables'] = tables success() return snapshot
def train_gan(train_set, indices: List, samples_per_N:int, repetition_n:int, identifier:str,experiment_name:str, batch_size: int = 256, desired_epochs: int = 2000, use_bot = False): """ The GAN is trained for 1000 epochs. If a a set of 60k samples is trained with a batchsize of 256, then a epoch equals 226 iterations. A budget of 100,000 iterations would equals to 426 """ assert train_set.shape[0] > len(indices) print(train_set.shape) print(len(indices)) my_ds = DataSetManager(train_set[indices]) # print("Set number of iterations to train\n") v5 = (desired_epochs*(train_set[indices].shape[0]))//batch_size +1 print("ITERS "+str(v5)) print("SIZE "+str(train_set[indices].shape)) # print("Use pretrained model? (0 means No, some number different to 0 means yes)\n") decision_number = 0 #int( input() ) # print("Type a name to save the model with?\n") model_tag = str(round(samples_per_N)) +'_'+ str(repetition_n) storing_path = 'data/'+ experiment_name + "/" + model_tag + '_data/' model_path = storing_path+ model_tag + '.ckpt' # Recall that os.mkdir isn't recursive, so it only makes on directoryt at a time try: # Create target Directory os.mkdir(storing_path) print("Directory " , storing_path , " Created ") except FileExistsError: print("Directory " , storing_path , " already exists") # ===> Auxiliar functions <=== """ ----------------8<-------------[ cut here ]------------------ ------------------------------------------------ """ def save_history(files_prefix, gen_loss_record,disc_loss_record, jsd_error, current_epoch, epoch_record,my_ds,iter_, epochs, global_iters, BATCH_SIZE, low_lr, high_lr ): # Save losses per epoch df = pd.DataFrame(np.array(gen_loss_record)) with open(files_prefix+'_gen_loss.csv', 'w+') as f: df.to_csv(f, header=False, index=False) df = pd.DataFrame(np.array(disc_loss_record)) with open(files_prefix+'_disc_loss.csv', 'w+') as f: df.to_csv(f, header=False, index=False) df = pd.DataFrame(np.array(epoch_record)) with open(files_prefix+'_epoch_record.csv', 'w+') as f: df.to_csv(f, header=False, index=False) # Save current iter and epochs training_history = {'epochs': [epochs + my_ds.epochs_completed], 'iters': [global_iters + iter_], 'Batch Size': [BATCH_SIZE], 'low LR': [low_lr], 'high LR': [high_lr]} df = pd.DataFrame(training_history) with open(files_prefix+'_training.csv', 'w+') as f: df.to_csv(f, index=False) #, header=False, index=False with open(files_prefix+'_jsd_error.csv', 'a') as csvFile: writer = csv.writer(csvFile) writer.writerow([current_epoch, jsd_error]) def send_bot_message(bot,my_ds, iter_, ITERS, identifier ): """ Not quite straighforward since the critic draws many more samples. """ message = "\nEpochs ["+str(my_ds.epochs_completed)+"] Iter: "+str(iter_)+";\t"+str(np.round(100* iter_/ITERS,2))+"% " message = message + identifier print(message) bot.set_status(message) # Send update message if bot.verbose: bot.send_message(message) print("\n") def save_gen_samples(gen_op, disc_op, sess,path, k, n = 4): """ k: is the number of epochs used to trained the generator n: is the number of batches to draw samples """ suffix = '_gen_samples_'+str(k)+'_epochs_'+'.csv' for k in range(n): samples = sess.run(gen_op) df = pd.DataFrame(np.array(samples)) with open(path+suffix, 'a') as f: df.to_csv(f, header=False, index=False) # Score the samples using the critic scores = sess.run(disc_op) df = pd.DataFrame(np.array(scores)) with open(path+'scores_'+suffix, 'a') as f: df.to_csv(f, header=False, index=False) # ===> Model Parameters <=== """ ----------------8<-------------[ cut here ]------------------ ------------------------------------------------ """ DIM = 512 # model dimensionality GEN_DIM = 100 # output dimension of the generator DIS_DIM = 1 # outptu dimension fo the discriminator FIXED_GENERATOR = False # wheter to hold the generator fixed at ral data plus Gaussian noise, as in the plots in the paper LAMBDA = .1 # smaller lambda makes things faster for toy tasks, but isn't necessary if you increase CRITIC_ITERS enough BATCH_SIZE = batch_size # batch size ITERS = v5 #100000 # how many generator iterations to train for FREQ = 250 # sample frequency print("==>>Using batch size of "+str(BATCH_SIZE)) CRITIC_ITERS = 5 # homw many critic iteractions per generator iteration def Generator_Softmax(n_samples, name='gen'): with tf.variable_scope(name): noise = tf.random_normal([n_samples, GEN_DIM]) output01 = tf_utils.linear(noise, 2*DIM, name='fc-1') output01 = tf_utils.relu(output01, name='relu-1') output02 = tf_utils.linear(output01, 2*DIM, name='fc-2') output02 = tf_utils.relu(output02, name='relu-2') output03 = tf_utils.linear(output02, 2*DIM, name='fc-3') output03 = tf_utils.relu(output03, name='relu-3') output04 = tf_utils.linear(output03, GEN_DIM, name='fc-4') # Reminder: a logit can be modeled as a linear function of the predictors output05 = tf.nn.softmax(output04, name = 'softmax-1') return output05 def Discriminator(inputs, is_reuse=True, name='disc'): with tf.variable_scope(name, reuse=is_reuse): print('is_reuse: {}'.format(is_reuse)) output01 = tf_utils.linear(inputs, 2*DIM, name='fc-1') output01 = tf_utils.relu(output01, name='relu-1') output02 = tf_utils.linear(output01, 2*DIM, name='fc-2') output02 = tf_utils.relu(output02, name='relu-2') output03 = tf_utils.linear(output02, 2*DIM, name='fc-3') output03 = tf_utils.relu(output03, name='relu-3') output04 = tf_utils.linear(output03, DIS_DIM, name='fc-4') return output04 real_data = tf.placeholder(tf.float32, shape=[None, GEN_DIM]) fake_data = Generator_Softmax(BATCH_SIZE) disc_real = Discriminator(real_data, is_reuse=False) disc_fake = Discriminator(fake_data) disc_cost = tf.reduce_mean(disc_fake) - tf.reduce_mean(disc_real) gen_cost = - tf.reduce_mean(disc_fake) # WGAN gradient penalty parameters alpha = tf.random_uniform(shape=[BATCH_SIZE, 1], minval=0., maxval=1.) interpolates = alpha*real_data + (1.-alpha) * fake_data disc_interpolates = Discriminator(interpolates) gradients = tf.gradients(disc_interpolates, [interpolates][0]) slopes = tf.sqrt(tf.reduce_sum(tf.square(gradients), reduction_indices=[1])) gradient_penalty = tf.reduce_mean((slopes - 1)**2) disc_cost += LAMBDA * gradient_penalty disc_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='disc') gen_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='gen') disc_lr = tf.placeholder(tf.float32, shape=()) # 1e-4 gen_lr = tf.placeholder(tf.float32, shape=()) # 1e-4 disc_train_op = tf.train.AdamOptimizer(learning_rate=disc_lr, beta1=0.5, beta2=0.9).minimize(disc_cost, var_list=disc_vars) if len(gen_vars) > 0: gen_train_op = tf.train.AdamOptimizer(learning_rate=gen_lr, beta1=0.5, beta2=0.9).minimize(gen_cost, var_list=gen_vars) else: gen_train_op = tf.no_op() """ ----------------8<-------------[ cut here ]------------------ ------------------------------------------------ """ # ===> Model Parameters <=== session_saver = tf.train.Saver() # files_prefix = 'model/'+ model_tag if decision_number == 0: pre_trained = False gen_loss_record = [] # type: List[float] disc_loss_record = [] # type: List[float] epoch_record = [] # type: List[float] epochs = 0 global_iters = 0 df = pd.DataFrame(np.array(indices)) with open(storing_path+'training_indices.csv', 'w+') as f: df.to_csv(f, header=False, index=False) else: pre_trained = True print(storing_path) print(storing_path+'training_indices.csv') _indices = (pd.read_csv(storing_path+'training_indices.csv',header=None ).values).tolist() print(len(_indices)) print(train_set[indices].shape) print(train_set[_indices].squeeze().shape) assert train_set[_indices].squeeze().shape == train_set[indices].shape my_ds = DataSetManager(train_set[_indices].squeeze()) temp = pd.read_csv(storing_path+'_training.csv',header=None ).values epochs, global_iters = temp.flatten() my_ds.epochs_completed = epochs gen_loss_record = (pd.read_csv(storing_path+'_gen_loss.csv',header=None ).values).tolist() disc_loss_record = (pd.read_csv(storing_path+'_disc_loss.csv',header=None ).values).tolist() epoch_record = (pd.read_csv(storing_path+'_epoch_record.csv',header=None ).values).tolist() print("State has been restored") # Create a DLBot instance if use_bot: bot = DLBot(token=telegram_token, user_id=telegram_user_id) # Activate the bot bot.activate_bot() print("\nTelegram bot has been activated ") iters_per_epoch = my_ds.num_examples/BATCH_SIZE total_iters = int(np.ceil((desired_epochs*iters_per_epoch)/CRITIC_ITERS)) critic_iters = np.round((5/6)*total_iters) gen_iters = np.round((1/6)*total_iters) ITERS = total_iters # Train loop with tf.Session() as sess: if pre_trained == False: # false by default: sess.run(tf.global_variables_initializer()) if pre_trained == True: session_saver.restore(sess,model_path) # # DUCK TAPE SOLUTION iter_ = 0 """ while my_ds.epochs_completed < desired_epochs: iter_ +=1 """ # r=10**-4.72, max_lr=10**-3.72, lr_multiplier :int = 1 low_lr = 10**-5 high_lr = 10**-4 lr1 = low_lr # lr_multiplier*low_lr lr2 = low_lr #lr_multiplier*high_lr gen_lr_ = low_lr # CyclicLR(base_lr= lr1, max_lr= lr2, step_size=gen_iters) disc_lr_ = low_lr # CyclicLR(base_lr= lr1, max_lr= lr2, step_size=critic_iters) for iter_ in range(ITERS): batch_data, disc_cost_ = None, None previous_epoch = my_ds.epochs_completed # train critic for i_ in range(CRITIC_ITERS): batch_data = my_ds.next_batch(BATCH_SIZE) # data_gen.__next__() disc_cost_, _ = sess.run([disc_cost, disc_train_op], feed_dict={real_data: batch_data, disc_lr:disc_lr_ }) # .clr() # disc_lr_.on_batch_end() # train generator sess.run(gen_train_op, feed_dict={gen_lr : gen_lr_}) # gen_lr_.clr() # gen_lr_.on_batch_end() gen_cost2 = sess.run(gen_cost) current_epoch = my_ds.epochs_completed condition2 = current_epoch % 5 == 0 if current_epoch > previous_epoch and condition2: disc_loss_record.append(disc_cost_) gen_loss_record.append(gen_cost2) epoch_record.append(my_ds.epochs_completed ) # print("Diff "+str(current_epoch - previous_epoch)) if (np.mod(iter_, FREQ) == 0) or (iter_+1 == ITERS): """ print("===> Debugging") print(disc_loss_record) print(gen_loss_record) """ if use_bot: bot.loss_hist.append(disc_cost_) fake_samples = sess.run(fake_data) # , feed_dict={real_data: batch_data} # print("\n==> Sum-Simplex condition: " +str(np.sum(fake_samples, axis=1))) fake_population = np.array([ sess.run(fake_data) for k in range(40)]).reshape(40*batch_size,train_set.shape[1]) print(fake_population.shape) jsd_error = gan_error_all_species(fake_population, k3_test_set) print("JSD Error "+str(jsd_error)) message = "\nEpochs ["+str(my_ds.epochs_completed)+"] Iter: "+str(iter_)+";\t"+str(np.round(100* iter_/ITERS,2))+"% " message = message + identifier print(message) if use_bot: send_bot_message(bot,my_ds, iter_, ITERS, identifier) current_epoch = my_ds.epochs_completed session_saver.save(sess, model_path) save_history(storing_path, gen_loss_record,disc_loss_record, jsd_error, current_epoch, epoch_record, my_ds,iter_, epochs, global_iters, BATCH_SIZE, low_lr, high_lr) # save_gen_samples(fake_data, disc_fake ,sess, storing_path, k) # fake_data = Generator_Softmax(BATCH_SIZE) utils.tick() # _iter[0] += 1 if iter_ == ITERS: session_saver.save(sess, model_path) # Create gan samples n_samples = len(indices) k_iter = n_samples//BATCH_SIZE +1 gan_samples_path = storing_path+"gan_samples_" +model_tag+'.csv' for k in range(k_iter): fake_samples = sess.run(fake_data) df = pd.DataFrame(fake_samples) with open(gan_samples_path, 'a') as f: df.to_csv(f, header=False, index=False) # Clear variables valuies tf.reset_default_graph() current_epoch = my_ds.epochs_completed save_history(storing_path, gen_loss_record,disc_loss_record, jsd_error, current_epoch, epoch_record, my_ds,iter_, epochs, global_iters, BATCH_SIZE, low_lr, high_lr) if use_bot: bot.stop_bot() print("Training is done") # Duct tapping the size of gan sample set to avoid changing the TF Graph temp1 = pd.read_csv(gan_samples_path, header=None).values temp1 = temp1[0:n_samples] df = pd.DataFrame(temp1) with open(gan_samples_path, 'w+') as f: df.to_csv(f, header=False, index=False) print("Training is done")
print "API get_orders error" exit() for i in ids: r = api.cancel_order(i, currency) if check(r): print "API cancel_order OK" else: print r["message"] print "API cancel_order error" a = float(account["result"]["balance"][currency.split("_")[0].upper()]["amount"]) b = float(account["result"]["balance"][currency.split("_")[-1].upper()]["amount"]) tick = utils.tick(currency) sell = float(tick["ticker"]["sell"]) buy = float(tick["ticker"]["buy"]) amount = "%.3f" % (b / sell, ) if float(amount) < 0.001: amount = "%.3f" % (a, ) price = buy * 1.1 t = 0 if float(amount) < 0.001: print "no money or btc for test" exit() else: price = sell * 0.9 t = 1
def train_gan(train_set, indices: List, samples_per_N: float, repetition_n: int): # print("Type a name to save the model with?\n") model_tag = str(round(samples_per_N)) + '_' + str(repetition_n) model_path = 'model/' + model_tag + '.ckpt' # print("Set number of iterations to train\n") v5 = 1000 # print("Use pretrained model? (0 means No, some number different to 0 means yes)\n") decision_number = 0 #int( input() ) storing_path = 'model/' + model_tag + '_data/' dirName = storing_path try: # Create target Directory os.mkdir(dirName) print("Directory ", dirName, " Created ") except FileExistsError: print("Directory ", dirName, " already exists") # ===> Auxiliar functions <=== """ ----------------8<-------------[ cut here ]------------------ ------------------------------------------------ """ def save_history(files_prefix, gen_loss_record, disc_loss_record, epoch_record, my_ds, iter_, epochs, global_iters): # Save losses per epoch df = pd.DataFrame(np.array(gen_loss_record)) with open(files_prefix + '_gen_loss.csv', 'w+') as f: df.to_csv(f, header=False, index=False) df = pd.DataFrame(np.array(disc_loss_record)) with open(files_prefix + '_disc_loss.csv', 'w+') as f: df.to_csv(f, header=False, index=False) df = pd.DataFrame(np.array(epoch_record)) with open(files_prefix + '_epoch_record.csv', 'w+') as f: df.to_csv(f, header=False, index=False) # Save current iter and epochs df = pd.DataFrame( np.array([epochs + my_ds.epochs_completed, global_iters + iter_])) with open(files_prefix + '_training.csv', 'w+') as f: df.to_csv(f, header=False, index=False) def send_bot_message(bot, my_ds, iter_, ITERS): message = "\nEpochs [" + str( my_ds.epochs_completed) + "] Iter: " + str(iter_) + " , % " + str( 100 * iter_ / ITERS) print(message) bot.set_status(message) # Send update message if bot.verbose: bot.send_message(message) print("\n") def save_gen_samples(gen_op, disc_op, sess, path, k, n=4): """ k: is the number of epochs used to trained the generator n: is the number of batches to draw samples """ suffix = '_gen_samples_' + str(k) + '_epochs_' + '.csv' for k in range(n): samples = sess.run(gen_op) df = pd.DataFrame(np.array(samples)) with open(path + suffix, 'a') as f: df.to_csv(f, header=False, index=False) # Score the samples using the critic scores = sess.run(disc_op) df = pd.DataFrame(np.array(scores)) with open(path + 'scores_' + suffix, 'a') as f: df.to_csv(f, header=False, index=False) # ===> Model Parameters <=== """ ----------------8<-------------[ cut here ]------------------ ------------------------------------------------ """ DIM = 512 # model dimensionality GEN_DIM = 100 # output dimension of the generator DIS_DIM = 1 # outptu dimension fo the discriminator FIXED_GENERATOR = False # wheter to hold the generator fixed at ral data plus Gaussian noise, as in the plots in the paper LAMBDA = .1 # smaller lambda makes things faster for toy tasks, but isn't necessary if you increase CRITIC_ITERS enough BATCH_SIZE = 256 # batch size ITERS = v5 #100000 # how many generator iterations to train for FREQ = 250 # sample frequency CRITIC_ITERS = 5 # homw many critic iteractions per generator iteration def Generator_Softmax(n_samples, name='gen'): with tf.variable_scope(name): noise = tf.random_normal([n_samples, GEN_DIM]) output01 = tf_utils.linear(noise, DIM, name='fc-1') output01 = tf_utils.relu(output01, name='relu-1') output02 = tf_utils.linear(output01, DIM, name='fc-2') output02 = tf_utils.relu(output02, name='relu-2') output03 = tf_utils.linear(output02, DIM, name='fc-3') output03 = tf_utils.relu(output03, name='relu-3') output04 = tf_utils.linear(output03, GEN_DIM, name='fc-4') # Reminder: a logit can be modeled as a linear function of the predictors output05 = tf.nn.softmax(output04, name='softmax-1') return output05 def Discriminator(inputs, is_reuse=True, name='disc'): with tf.variable_scope(name, reuse=is_reuse): print('is_reuse: {}'.format(is_reuse)) output01 = tf_utils.linear(inputs, DIM, name='fc-1') output01 = tf_utils.relu(output01, name='relu-1') output02 = tf_utils.linear(output01, DIM, name='fc-2') output02 = tf_utils.relu(output02, name='relu-2') output03 = tf_utils.linear(output02, DIM, name='fc-3') output03 = tf_utils.relu(output03, name='relu-3') output04 = tf_utils.linear(output03, DIS_DIM, name='fc-4') return output04 real_data = tf.placeholder(tf.float32, shape=[None, GEN_DIM]) fake_data = Generator_Softmax(BATCH_SIZE) disc_real = Discriminator(real_data, is_reuse=False) disc_fake = Discriminator(fake_data) disc_cost = tf.reduce_mean(disc_fake) - tf.reduce_mean(disc_real) gen_cost = -tf.reduce_mean(disc_fake) # WGAN gradient penalty parameters alpha = tf.random_uniform(shape=[BATCH_SIZE, 1], minval=0., maxval=1.) interpolates = alpha * real_data + (1. - alpha) * fake_data disc_interpolates = Discriminator(interpolates) gradients = tf.gradients(disc_interpolates, [interpolates][0]) slopes = tf.sqrt(tf.reduce_sum(tf.square(gradients), reduction_indices=[1])) gradient_penalty = tf.reduce_mean((slopes - 1)**2) disc_cost += LAMBDA * gradient_penalty disc_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='disc') gen_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='gen') #WGAN Training operations disc_train_op = tf.train.AdamOptimizer(learning_rate=1e-4, beta1=0.5, beta2=0.9).minimize( disc_cost, var_list=disc_vars) if len(gen_vars) > 0: gen_train_op = tf.train.AdamOptimizer(learning_rate=1e-4, beta1=0.5, beta2=0.9).minimize( gen_cost, var_list=gen_vars) else: gen_train_op = tf.no_op() """ ----------------8<-------------[ cut here ]------------------ ------------------------------------------------ """ # ===> Model Parameters <=== print("==> Loading CSV") v1 = 'data/' + str(archivero) print(v1) train_data = pd.read_csv( v1, header=None ).values # np.random.normal(mu, sigma, (n_train_samples,numero_especies)) mean_vec = np.zeros(100) # vector de 100 ceros print("Shape") print(train_data.shape) print("One samples mean") print(np.mean(train_data[0, :])) my_ds = DataSetManager(train_data, norm=False) session_saver = tf.train.Saver() files_prefix = 'model/' + model_tag if decision_number == 0: pre_trained = False gen_loss_record = [] # type: List[float] disc_loss_record = [] # type: List[float] epoch_record = [] # type: List[float] epochs = 0 global_iters = 0 else: pre_trained = True temp = pd.read_csv(storing_path + '_training.csv', header=None).values epochs, global_iters = temp.flatten() my_ds.epochs_completed = epochs gen_loss_record = (pd.read_csv(storing_path + '_gen_loss.csv', header=None).values).tolist() disc_loss_record = (pd.read_csv(storing_path + '_disc_loss.csv', header=None).values).tolist() epoch_record = (pd.read_csv(storing_path + '_epoch_record.csv', header=None).values).tolist() # Create a DLBot instance bot = DLBot(token=telegram_token, user_id=telegram_user_id) # Activate the bot bot.activate_bot() print("\nTelegram bot has been activated ") # Train loop with tf.Session() as sess: if pre_trained == False: # false by default: sess.run(tf.global_variables_initializer()) if pre_trained == True: # tf.reset_default_graph() session_saver.restore(sess, model_path) for iter_ in range(ITERS): batch_data, disc_cost_ = None, None previous_epoch = my_ds.epochs_completed # train critic for i_ in range(CRITIC_ITERS): batch_data = my_ds.next_batch( BATCH_SIZE) # data_gen.__next__() disc_cost_, _ = sess.run([disc_cost, disc_train_op], feed_dict={real_data: batch_data}) # train generator sess.run(gen_train_op) gen_cost2 = sess.run(gen_cost) current_epoch = my_ds.epochs_completed condition2 = current_epoch % 5 == 0 if current_epoch > previous_epoch and condition2: disc_loss_record.append(disc_cost_) gen_loss_record.append(gen_cost2) epoch_record.append(my_ds.epochs_completed) # print("Diff "+str(current_epoch - previous_epoch)) if (np.mod(iter_, FREQ) == 0) or (iter_ + 1 == ITERS): """ print("===> Debugging") print(disc_loss_record) print(gen_loss_record) """ bot.loss_hist.append(disc_cost_) fake_samples = sess.run( fake_data) # , feed_dict={real_data: batch_data} # print("\n==> Sum-Simplex condition: " +str(np.sum(fake_samples, axis=1))) send_bot_message(bot, my_ds, iter_, ITERS) session_saver.save(sess, model_path) save_history(storing_path, gen_loss_record, disc_loss_record, epoch_record, my_ds, iter_, epochs, global_iters) k = my_ds.epochs_completed save_gen_samples( fake_data, disc_fake, sess, storing_path, k) # fake_data = Generator_Softmax(BATCH_SIZE) utils.tick() # _iter[0] += 1 if iter_ == ITERS: session_saver.save(sess, model_path) save_history(storing_path, gen_loss_record, disc_loss_record, epoch_record, my_ds, iter_, epochs, global_iters) k = my_ds.epochs_completed bot.stop_bot() print("Training is done") n_samples = len(indices) print("Training is done")
com = [] hom = [] chi = [] ticksize, tickmarks = tickinit(nbr_runs) chrono() for run_nbr in range(nbr_runs): km = KMeans(n_clusters=k, random_state=run_nbr).fit(X_clust) labs = km.labels_ sil.append(silhouette_score(X_clust, labs)) ari.append(adjusted_rand_score(yyy, labs)) ami.append(adjusted_mutual_info_score(yyy,labs)) com.append(completeness_score(yyy,labs)) hom.append(homogeneity_score(yyy,labs)) chi.append(calinski_harabasz_score(X_clust, labs)) tick(ticksize, tickmarks, run_nbr) print("│ {0:.2f}s\n".format(chrono())) min_sil.append(np.amin(sil)) avg_sil.append(sum(sil)/len(sil)) max_sil.append(np.amax(sil)) min_ari.append(np.amin(ari)) avg_ari.append(sum(ari)/len(ari)) max_ari.append(np.amax(ari)) min_ami.append(np.amin(ami)) avg_ami.append(sum(ami)/len(ami)) max_ami.append(np.amax(ami)) min_com.append(np.amin(com))
def main(): global verbose # Parsing arguments parser = argparse.ArgumentParser(description='Train a neural network.') get_args(parser) args = parser.parse_args() if args.verbose: utils.verbose = True machine_learning.model_param['verbose'] = True if args.loss: machine_learning.model_param['loss'] = args.loss if args.optimization: machine_learning.model_param['optimizer'] = args.optimization if args.epochs: machine_learning.model_param['nb_epoch'] = args.epochs if args.batch_size: machine_learning.model_param['batch_size'] = args.batch_size # Training set print "######################################################################" print " Reading training set..." utils.tick() train_input, train_output, descriptor_labels, document_labels = utils.load_input_and_output( args.train_input, args.train_output, args.train_restrict, args.train_exclude) if utils.verbose: print "-> ", len(train_input), "entries loaded" print "Example of input:\t", train_input[0] print "Example of output:\t", train_output[0] utils.tock() # Dev set if args.dev_input and args.dev_output: print "######################################################################" print " Reading dev set..." utils.tick() dev_input, dev_output, _, _ = utils.load_input_and_output( args.dev_input, args.dev_output, args.dev_restrict, args.dev_exclude) if utils.verbose: print "-> ", len(dev_input), "entries loaded" print "Example of input:\t", dev_input[0] print "Example of output:\t", dev_output[0] utils.tock() # Test set if args.test_input and args.test_output: print "######################################################################" print " Reading test set..." utils.tick() test_input, test_output, _, _ = utils.load_input_and_output( args.test_input, args.test_output, args.test_restrict, args.test_exclude) if utils.verbose: print "-> ", len(test_input), "entries loaded" print "Example of input:\t", test_input[0] print "Example of output:\t", test_output[0] utils.tock() print "######################################################################" print "Input dimension =", len(train_input[0]) # Train model utils.tick() print "######################################################################" print "Training... " if args.dev_input and args.dev_output: model = machine_learning.train_neural_network(args.save_model, train_input, train_output, dev_input, dev_output) else: model = machine_learning.train_neural_network(args.save_model, train_input, train_output) utils.tock() if test_input and test_output: # Test model utils.tick() print "######################################################################" print "Testing... " machine_learning.evaluate(model, test_input, test_output) utils.tock()
bot.loss_hist.append(disc_cost_) fake_samples = sess.run( fake_data) # , feed_dict={real_data: batch_data} # print("\n==> Sum-Simplex condition: " +str(np.sum(fake_samples, axis=1))) send_bot_message(bot, my_ds, iter_, ITERS) session_saver.save(sess, model_path) save_history(storing_path, gen_loss_record, disc_loss_record, epoch_record, my_ds, iter_, epochs, global_iters) k = my_ds.epochs_completed save_gen_samples(fake_data, disc_fake, sess, storing_path, k) # fake_data = Generator_Softmax(BATCH_SIZE) utils.tick() # _iter[0] += 1 if iter_ == ITERS: session_saver.save(sess, model_path) save_history(storing_path, gen_loss_record, disc_loss_record, epoch_record, my_ds, iter_, epochs, global_iters) k = my_ds.epochs_completed bot.stop_bot() print("Training is done") print("Training is done")
print("==> Values Greater than mu="+str(mu)+"; n="+str(np.sum(fake_samples > mu))) a2 = str(np.mean(fake_samples)) print("==> Mean: "+a2) a3 = str(np.std(fake_samples)) print("==> STD: "+a3) utils.flush(img_folder) # generate_image(sess, batch_data, iter_) if (np.mod(iter_, FREQ) == 0) or (iter_+1 == ITERS): session_saver.save(sess, 'model/my_gan.ckpt') utils.tick() # In[12]: np.sum(fake_samples -1 < 0) # In[13]: np.std(fake_samples)
def real_buy(): access_key = config.access_key access_secret = config.access_secret chbtc = api.chbtcApi(access_key, access_secret) account = chbtc.get_account_info() currency = "btc_cny" a = float( account["result"]["balance"][currency.split("_")[0].upper()]["amount"]) b = float(account["result"]["balance"][currency.split("_")[-1].upper()] ["amount"]) ticker = utils.tick(currency) tbuy, tsell = float(ticker["ticker"]["buy"]), float( ticker["ticker"]["sell"]) if float(b) / tsell < 0.001: print "out of money" return False conn = sqlite3.connect("%s/%s" % (config.db_dir, config.db_file)) c = conn.cursor() k = {} digests = [i[0] for i in rank() if i[1] > 0] c.execute( "select digest, content, currency, t, size, n, increase, feature_size, point from features where digest in (%s)order by point" % (",".join('?' * len(digests)), ), digests) for f in c.fetchall(): digest, content, currency, t, size, n, increase, feature_size, point = f if "%s_%s_%s" % (currency, t, size) not in k: k["%s_%s_%s" % (currency, t, size)] = utils.kline( currency, t, "", size) print "Real comparing to %s @ %s %s %s" % ( digest, datetime.datetime.fromtimestamp( point / 1000).strftime('%Y-%m-%d %H:%M:%S'), increase, t) s = kline_similarity(k["%s_%s_%s" % (currency, t, size)], json.loads(content), feature_size) v = float(json.loads(content)["v"]) if not math.isnan(s) and s <= v: #if True: created_at = int(time.time() * 1000) c.execute( "select * from trade where digest = ? and created_at > ? and updated_at = 0", (digest, created_at - 1000 * n * utils.str2sec(t))) if len(c.fetchall()) > 0: continue print "Real match" r = trade(currency, "b", 1.0) print r if r != False: #ticker = utils.tick(currency) #buy, sell = float(ticker["ticker"]["buy"]), float(ticker["ticker"]["sell"]) print "real buy %s @ %s, amount %s" % (currency, r["price"], r['total_amount']) c.execute( "Insert into trade (created_at, updated_at, digest, buy, sell) values (?, 0, ?, ?, 0.0)", (r["trade_date"], digest, r["price"])) conn.commit() else: #print "Not Match" pass conn.close()