def __call__(self): self.initialize() with tf.Graph().as_default(): self.build_graph() self.build_saver() with create_session() as sess: self.run(sess)
def login_route(request): if request.Method == 'GET': header = 'HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nConnection:Close\r\n' verify_session = request.cookies().get('username', '') username = session.get(verify_session, '') user = User.find_by(username) if user is not None: return redirect('/') else: body = template('login.html') r = header + '\r\n' + body return r if request.Method == 'POST': re_username = request.form().get('username', '') re_password = request.form().get('password', '') user = User.login_verify(re_username, re_password) if user: """如果登录成功,获取一个随机16位的字符串,和username一起放入session字典中""" user_session = create_session() session[user_session] = user.username return redirect('/', user_session) else: print '登录失败' my_word = '登录失败' header = 'HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nConnection:Close\r\n' body = template( 'login_success.html', my_word=my_word, ) r = header + '\r\n' + body return r
def init(self): import tensorflow as tf self.env = self.env_producer.get_new_environment() self.s0 = self.env.reset() self.session = utils.create_session(self.env_opts, False) with tf.device("/cpu:0"): with tf.variable_scope("gather-%s" % self.idx): pol = get_policy(self.env_opts, self.session) self.agent = PPOAgent(pol, self.session, "gather-%s" % self.idx, self.env_opts) self.trainable_vars = tf.get_collection( tf.GraphKeys.TRAINABLE_VARIABLES, "gather-%s" % self.idx) self.accum_vars = [ tf.Variable(tf.zeros_like(tv.initialized_value()), trainable=False) for tv in self.trainable_vars ] assign_ops = [ self.trainable_vars[i].assign(self.accum_vars[i]) for i in range(len(self.trainable_vars)) ] self.assign_op = tf.group(assign_ops) self.session.run(tf.global_variables_initializer()) self.cur_hidden_state = self.agent.get_init_hidden_state() self.episode = [self.s0], [], [], [], [], [self.cur_hidden_state ], []
def evaluate(paths, data, params): total = {} for path in paths: model = load_model(path, params) with utils.create_session() as sess: model.saver.restore(sess, '%s/model' % model.path) sess.run([model.valid_dataset.initializer], feed_dict={ model.valid_dataset.x: data.x, model.valid_dataset.y: data.y }) accuracy, probabilities, labels = validate(sess, model) if 'probabilities' not in total: total['probabilities'] = probabilities total['labels'] = labels else: total['probabilities'] += probabilities print("%s: " % model.name, accuracy) num_evaluated = total['probabilities'].shape[0] predictions = np.argmax(total['probabilities'], axis=1) num_correct = 0.0 + np.sum(predictions == total['labels']) accuracy = num_correct / num_evaluated print("Accuracy (%d models): %.3f" % (len(paths), accuracy))
def create_session(self): self.train_writer = tf.summary.FileWriter(self.train_dir + '/train', tf.get_default_graph(), max_queue=20, flush_secs=120) self.val_writer = tf.summary.FileWriter(self.train_dir + '/val') return create_session(debug=self.debug)
def sampler_on_nd_gaussian(burnin, num_chains, num_samples, dim): # define the gaussian np.random.seed(666) mu = np.random.uniform(0, 20, dim).astype(np.float32) cov = np.random.uniform(0, 10, (dim, dim)).astype(np.float32) cov = np.dot(cov, cov.T) cov = cov / cov.max() # cov = np.identity(dim).astype(np.float32) cov = (cov + cov.T) / 2. cov[np.arange(dim), np.arange(dim)] = 1.0 cov_inv = np.linalg.inv(cov) def gaussian_energy(x): return 0.5 * tf.reduce_sum( tf.multiply(tf.matmul(x - mu, cov_inv), x - mu), 1) init_pos = np.random.normal(size=(num_chains, dim)) hmc_sampler = HamiltonianSampler(init_pos, gaussian_energy, init_stepsize=1, stepsize_max=5, avg_accept_slowness=0.9) sample_op, updates = hmc_sampler.sample() samples = [] sess = utils.create_session() with sess.as_default(): tf.set_random_seed(666) tf.global_variables_initializer().run() for _ in range(burnin): sess.run([sample_op, updates]) for i in range(num_samples): new_sample, _ = sess.run([sample_op, updates]) samples.append(new_sample) final_stepsize = sess.run(hmc_sampler.stepsize) final_accept_rate = sess.run(hmc_sampler.avg_accept_rate) samples = np.array(samples) print samples.shape samples = samples.T.reshape(dim, -1).T print samples.shape print '****** TARGET VALUES ******' print 'target mean:', mu print 'target cov:\n', cov print '****** EMPIRICAL MEAN/COV USING HMC ******' print 'empirical mean: ', samples.mean(axis=0) print 'empirical_cov:\n', np.cov(samples.T) print '****** HMC INTERNALS ******' print 'final stepsize:', final_stepsize print 'final acceptance_rate', final_accept_rate print 'DIFF' print np.abs(cov - np.cov(samples.T)).sum() print cov.sum() - np.cov(samples.T).sum()
def __call__(self): self.initialize() self.get_dataset() with tf.Graph().as_default(): self.build_graph() self.build_saver() with create_session() as sess: self.run_last(sess) self.run_steps(sess)
def main(): schema_name = 'etl' schema_create_session = create_session(engine) create_new_schema(schema_name=schema_name, session=schema_create_session) for data_file in glob.glob("./data/FDAData/*.csv"): print(os.path.basename(data_file)) send_data_to_postgres(data_file=data_file, schema_name=schema_name)
def init_agent(self): import tensorflow as tf env_opts = environments.get_env_options( self.env_name, self.env_producer.get_use_gpu()) self.session = utils.create_session(env_opts, True) with tf.variable_scope("worker-%s" % self.idx): pol = get_policy(env_opts, self.session) self.agent = PPOAgent(pol, self.session, "worker-%s" % self.idx, env_opts) self.trainable_vars = tf.get_collection( tf.GraphKeys.TRAINABLE_VARIABLES, "worker-%s" % self.idx) self.accum_vars = [ tf.Variable(tf.zeros_like(tv.initialized_value()), trainable=False) for tv in self.trainable_vars ] p_vars = self.agent.p_opt.variables() v_vars = self.agent.v_opt.variables() self.p_opt_vars = [ tf.Variable(tf.zeros_like(tv.initialized_value()), trainable=False) for tv in p_vars ] self.v_opt_vars = [ tf.Variable(tf.zeros_like(tv.initialized_value()), trainable=False) for tv in v_vars ] p_assign_ops = [ p_vars[i].assign(self.p_opt_vars[i]) for i in range(len(p_vars)) ] v_assign_ops = [ v_vars[i].assign(self.v_opt_vars[i]) for i in range(len(v_vars)) ] assign_ops = [ self.trainable_vars[i].assign(self.accum_vars[i]) for i in range(len(self.trainable_vars)) ] self.assign_op = tf.group(assign_ops + p_assign_ops + v_assign_ops) self.session.run(tf.global_variables_initializer()) self.run()
def start(): print('Consulta Saldo Transurc\n') session = create_session() # first_response = send_first_request(session) # second_url = find_second_url(first_response) # second_response = send_second_request(session, second_url) # Create value of the header 'Cookie' necessary to the # third request and last request cookie_value = make_cookie_value(session) add_new_header(config.HEADERS_3, 'Cookie', cookie_value) add_new_header(config.HEADERS_4, 'Cookie', cookie_value) third_response = send_third_request(session, config.THIRD_URL) # save_file(third_response, 'consulta_saldo_form_file.html') # user_data = { # 'num_aplicacao': input('Digite o número antes do cartão (XX): '), # 'num_cartao': input('Digite o número do seu cartão (XX): '), # 'digito_verificador': input('Digite o número verificador (X): '), # 'data_nascimento': input( # 'Digite a sua data de nascimento (DD/MM/AAAA):' # ) # } user_data = { 'num_aplicacao': sys.argv[1], 'num_cartao': sys.argv[2], 'digito_verificador': sys.argv[3], 'data_nascimento': sys.argv[4], } # Last response contains the balance last_response = send_last_request(session, third_response, user_data) balance = get_balance(last_response) print('\nSeu saldo no bilhete único é ' + balance)
if __name__ == '__main__': from dataset_wrapper import Cifar10Wrapper from rbm import RBM from autoencoder import AutoEncoder from dem_trainer import DEMTrainer import cifar10_ae import gibbs_sampler import utils import keras.backend as K import os import h5py import numpy as np np.random.seed(66699) sess = utils.create_session() K.set_session(sess) dataset = Cifar10Wrapper.load_default() ae_folder = 'prod/cifar10_ae3_relu_6/' encoder_weights_file = os.path.join(ae_folder, 'encoder.h5') decoder_weights_file = os.path.join(ae_folder, 'decoder.h5') rbm_params_file = os.path.join( ae_folder, 'ptrbm_scheme1/ptrbm_hid2000_lr0.001_pcd25/epoch_500_rbm.h5') # encoder_weights_file = '/home/hhu/Developer/dem/prod/cifar10_ae3_relu_6/test_ae_fe_const_balance/epoch_500_encoder.h5' # decoder_weights_file = encoder_weights_file.replace('encoder.', 'decoder.') # rbm_params_file = encoder_weights_file.replace('encoder.', 'rbm.') dem = DEM.load_from_param_files(dataset.x_shape, cifar10_ae.RELU_MAX,
import sys from csvparser import CSVParser # from models import Table from utils import create_session session = create_session('db.sqlite3') FILE_NAME = sys.argv[1] with open(FILE_NAME, 'r', encoding='utf8') as csv_file: csv_parser = CSVParser(csv_file, session=session) csv_parser.parse() print('Done!')
PROT = "file://" ROOTDIR = '<PATH DOVE SALVI I FILE>' INPUT = '<PATH DATASET INPUT>' # INPUT = '<PATH DATASET INPUT TEST (più piccolo)'> # TMPDIR = PATH DOVE SALVARE L'RDD AS TEXT FILE (GENERA UNA DIRECTORY) TMPDIR = '<PATH STESSO NOME OUTPUT + "-tmp">' OUTPUT = '<PATH DATASET OUTPUT FINALE>' def run_job(rdd): # rdd = rdd \ # OPERAZIONI SU RDD # .saveAsSequenceFile(TMPDIR) pass if __name__ == "__main__": spark = utils.create_session("<FB_JOB-ID>") sc = spark.sparkContext # rimuovo l'output del precedente job (genera errore se il file è protetto da scrittura, # serve a evitare di accorgersene all'ultimo) if os.path.exists(OUTPUT): os.remove(OUTPUT) # se ho interrotto il job precedente o sto rilanciando, rimuovo la directory di output if os.path.exists(TMPDIR): shutil.rmtree(TMPDIR) # RUNNO IL JOB (spark ha bisogno del protocollo file per caricare l'input) rdd = utils.load_data(sc, PROT + INPUT) run_job(rdd)
onto.append( format_triple(URI(FBO + uprop), RDFS.label, Literal(pretty_label(pred[2])))) return onto def run_job(rdd): rdd = rdd \ .map(generate_ontology) \ .flatMap(lambda x: x) \ .distinct() \ .repartition(1) \ .saveAsTextFile(TMPDIR) if __name__ == "__main__": spark = utils.create_session("FB_S5_ONTOLOGY") sc = spark.sparkContext if os.path.exists(OUTPUT): os.remove(OUTPUT) if os.path.exists(TMPDIR): shutil.rmtree(TMPDIR) fb_rdd = utils.load_data(sc, PROT + INPUT) run_job(fb_rdd) shutil.move(f"{TMPDIR}/part-00000", OUTPUT) shutil.rmtree(TMPDIR)
def clean_triple(row): # Shorten URL prefixes for /ns/ and /key/ namespaces row = row.replace(NS_PREFIX, NEW_NS_PREFIX).replace(KEY_PREFIX, NEW_KEY_PREFIX) # Remove CustomDataTypes from Literals (like w3.org's XMLSchema) return re.sub(r"\^\^[^\t]*", "", row) def run_job(rdd): rdd = rdd \ .filter(lambda x: not re.findall(SKIP_PATTERNS, x)) \ .map(clean_triple) \ .repartition(1) \ .saveAsTextFile(TMPDIR) if __name__ == "__main__": spark = utils.create_session("FB_S0") sc = spark.sparkContext if os.path.exists(OUTPUT): os.remove(OUTPUT) if os.path.exists(TMPDIR): shutil.rmtree(TMPDIR) fb_rdd = utils.load_data(sc, PROT + INPUT) run_job(fb_rdd) shutil.move(f"{TMPDIR}/part-00000", OUTPUT) shutil.rmtree(TMPDIR)
border_mode='same', subsample=(2,2))(x) x = BN(mode=2, axis=3)(x) # 14, 14, 20 x = Deconv2D(1, 3, 3, output_shape=[batch_size, 28, 28, 1], activation='sigmoid', border_mode='same', subsample=(2,2))(x) # 28, 28, 1 return x if __name__ == '__main__': import keras.backend as K from autoencoder import AutoEncoder from dataset_wrapper import MnistWrapper import utils K.set_session(utils.create_session()) mnist_dataset = MnistWrapper.load_default() # ----------normal relu pretraining---------- print 'Training model with normal relu' folder = 'test/mnist_ae_relu_inf' ae = AutoEncoder(mnist_dataset, encode, decode, None, folder) ae.build_models() num_epoch = 30 lr_schedule = utils.generate_decay_lr_schedule(num_epoch, 0.1, 1) ae.train(128, num_epoch, lr_schedule) ae.save_models() ae.test_models(utils.vis_mnist) ae.log()
import glob import os import pandas as pd import sqlalchemy.sql as sql from utils import create_new_schema, create_session, tyler_local_connect path_to_app_settings = os.path.abspath('appsettings.json') engine = tyler_local_connect(path_to_app_settings) session = create_session(engine) def schema_create_test(schema_name='etl'): query_results = create_new_schema(schema_name=schema_name, session=session, debug=True) print(query_results.scalar()) def test_session_location(): # query = sql.select([sql.text('database_name()')]) # return session.query(query).all() print(engine.table_names()) print(engine.schema_names()) if __name__ == '__main__': schema_create_test() print(test_session_location())
# utils.plot_images(data.test.images[:9], img_shape, data.test.labels[:9]) x = tf.placeholder(tf.float32, shape=[None, img_size_flat], name='x') x_image = tf.reshape(x, [-1, img_size, img_size, num_channels]) y_true = tf.placeholder(tf.float32, shape=[None, num_classes], name='y_true') y_true_class = tf.argmax(y_true, axis=1, output_type=tf.int32) model = x_image model = model * 2. - 1. # normalize max = tf.reduce_max(model) min = tf.reduce_min(model) mean = tf.reduce_mean(model) with utils.create_session() as sess: x_data, y_data = data.test.next_batch(test_batch_size) min, mean, max = sess.run([min, mean, max], feed_dict={ x: x_data, y_true: y_data }) print( "input_data_format:\n\t\tmin: {:.2f} mean: {:.2f} max: {:.2f}".format( min, mean, max)) model = layers.conv2d(inputs=model, name="conv1", padding='same', filters=32, kernel_size=3,
def start(self): import tensorflow as tf self.summary_writer = tf.summary.FileWriter("logs/%s" % self.env_opts["env_name"]) self.session = utils.create_session(self.env_opts, True) with tf.variable_scope("master-0"): pol = get_policy(self.env_opts, self.session) self.agent = PPOAgent(pol, self.session, "master-0", self.env_opts) self.trainable_vars = tf.get_collection( tf.GraphKeys.TRAINABLE_VARIABLES, "master-0") self.accum_vars = [ tf.Variable(tf.zeros_like(tv.initialized_value()), trainable=False) for tv in self.trainable_vars ] p_vars = self.agent.p_opt.variables() v_vars = self.agent.v_opt.variables() self.p_opt_vars = [ tf.Variable(tf.zeros_like(tv.initialized_value()), trainable=False) for tv in p_vars ] self.v_opt_vars = [ tf.Variable(tf.zeros_like(tv.initialized_value()), trainable=False) for tv in v_vars ] p_assign_ops = [ p_vars[i].assign(self.p_opt_vars[i]) for i in range(len(p_vars)) ] v_assign_ops = [ v_vars[i].assign(self.v_opt_vars[i]) for i in range(len(v_vars)) ] assign_ops = [ self.trainable_vars[i].assign(self.accum_vars[i]) for i in range(len(self.trainable_vars)) ] self.assign_op = tf.group(assign_ops + p_assign_ops + v_assign_ops) self.restore_variables() self.saver = tf.train.Saver(max_to_keep=1) self.session.run(tf.global_variables_initializer()) try: self.saver = tf.train.import_meta_graph( tf.train.latest_checkpoint("models/%s/" % self.env_opts["env_name"]) + ".meta") self.saver.restore( self.session, tf.train.latest_checkpoint("models/%s/" % self.env_opts["env_name"])) except: print("failed to restore model") while True: if self.iter_count % 10 == 0: print("Saving model...") self.save_variables() self.saver.save(self.session, self.model_path, self.iter_count) print("Model saved") self.broadcast_weights() self.merge_weights() self.iter_count += 1
.groupByKey() \ .mapValues(iterate) \ .repartition(1) \ .saveAsSequenceFile(TMPDIR) shutil.move(TMPDIR + "/part-00000", OUTPUT + '-tmp') shutil.rmtree(TMPDIR) with open(OUTPUT + '.json', 'w') as output: temp = sc.sequenceFile(OUTPUT + '-tmp').collectAsMap() json.dump(temp, output) os.remove(OUTPUT + '-tmp') if __name__ == "__main__": spark = utils.create_session("FB_dictionary_builder") sc = spark.sparkContext try: shutil.rmtree(OUTPUT + '.json') except: pass try: shutil.rmtree(TMPDIR) except: pass rdd = utils.load_data(sc, PROT + INPUT) run_job(rdd, sc)
def train(paths, params, retries_left=retries): build_new_model = False if not paths: build_new_model = True paths.append('...') # hack for path in paths: model = build_model(params) if build_new_model else load_model( path, params) with utils.create_session() as sess: vars_to_run = [ model.train_dataset.initializer, model.valid_dataset.initializer ] if build_new_model: vars_to_run += [ tf.variables_initializer( tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)) ] else: model.saver.restore(sess, '%s/model' % model.path) sess.run(vars_to_run, feed_dict={ model.train_dataset.x: data['train'].x, model.train_dataset.y: data['train'].y, model.valid_dataset.x: data['valid'].x, model.valid_dataset.y: data['valid'].y }) average = { "loss": 1, "accuracy": 0.5, "num_correct": params.batch_size / 2 } tr_epochs = trange(epochs) early_exitted = False # Training. for epoch in tr_epochs: num_batches = data[ 'train'].num_examples // params.batch_size + 1 tr_batch = trange(num_batches, leave=False) tr_batch.set_description('E %d/%d' % (epoch + 1, epochs)) if (epoch + 1) % swap_epochs_every == 0: data['train'].random_swaps() for batch_id in tr_batch: _, loss, num_correct, summaries, lr = sess.run( [ model.opt, model.total_loss, model.correct_predictions, model.all_summaries, model.lr ], feed_dict={model.dropout: params.dropout}) model.writer.add_summary(summaries, epoch * num_batches + batch_id) average['loss'] = utils.move_average(average['loss'], loss) average['num_correct'] = utils.move_average( average['num_correct'], num_correct) tr_batch.set_postfix(loss=average['loss'], accuracy=average['num_correct'] / params.batch_size) accuracy, _, _ = validate(sess, model) average['accuracy'] = utils.move_average( average['accuracy'], accuracy) global best_accuracy if accuracy > best_accuracy: best_accuracy = accuracy save_model(sess, model) tr_epochs.set_postfix(lr=lr, best_accuracy=best_accuracy, validation_accuracy=accuracy) if epoch >= epochs_early_threshold and average[ 'accuracy'] < accuracy_early_threshold: early_exitted = True print( "Low avg. accuracy: %.3f after %d epochs. Stopping early. Retries left: %d" % (average['accuracy'], epochs_early_threshold, retries_left)) break if early_exitted and retries_left > 0: return train(model_path, params, retries_left - 1)
OUTPUT = ROOTDIR + 'e2-rdfs_range' # override this to enable caching Data USE_CACHE = False CACHED_DATA_TMP = ROOTDIR + 'cached-ontology-tmp' CACHED_DATA = ROOTDIR + 'cached-ontology' LOOKUP_PRED = 'type.property.expected_type' OUTPUT_PRED = RDFS.range # regex for key filtering (select only properties) IS_PROPERTY = r"^<fbo:([^>\.]+\.){2}[^>\.]+>$" EXT_KEY_MAPPING = {'pattern': r"^<f:", 'replace': r"<fbo:"} if __name__ == "__main__": spark = utils.create_session("FB_S5_E2") sc = spark.sparkContext if os.path.exists(OUTPUT): os.remove(OUTPUT) if os.path.exists(TMPDIR): shutil.rmtree(TMPDIR) if USE_CACHE and os.path.exists(CACHED_DATA): data_rdd = utils.load_data(sc, PROT + CACHED_DATA) else: data_rdd = utils.load_data(sc, PROT + INPUT_DATA) ext_rdd = utils.load_data(sc, PROT + INPUT_EXT) results = extractor.run(data_rdd, ext_rdd, LOOKUP_PRED, OUTPUT_PRED,
utils.combine_growth_accs) \ .mapValues(utils.calculate_growth) \ .cache() min_max_rdd = minprice_rdd.join(maxprice_rdd) min_max_avgvol_rdd = min_max_rdd.join(avgvol_rdd) \ .mapValues(lambda value: (value[0][0], value[0][1], value[1])) metrics_rdd = min_max_avgvol_rdd.join(growth_rdd) \ .mapValues(lambda value: Row(growth=value[1], min_price=value[0][0], max_price=value[0][1], avg_volume=value[0][2], )) \ .cache() ranked_rdd = metrics_rdd.sortBy(lambda row: row[1].growth, ascending=False) \ .mapValues(pretty_print) \ .take(10) for k, v in ranked_rdd: print(k, v) if __name__ == "__main__": spark = utils.create_session("job1") sc = spark.sparkContext # sqlContext = SQLContext(sc) history_rdd = utils.load_data(spark, HISTORY_PATH) run_job(history_rdd)
def __init__(self): self.session = utils.create_session() self.__load_subject_dict() print('初始化 搜索器 %s 成功' % (__class__, ))
if d: s = env.reset() return rewards if __name__ == '__main__': logger.configure(f'{C.env_id}/logs_{time_stamp}') for k, v in C._asdict().items(): logger.record_tabular(k, v) logger.dump_tabular() max_reward = tf.placeholder(tf.float32, name='max_reward') mean_reward = tf.placeholder(tf.float32, name='mean_reward') max_summary = tf.summary.scalar('max_rew', max_reward) mean_summary = tf.summary.scalar('mean_rew', mean_reward) with create_session(0) as sess: eval_env = make_atari(C.env_id, 113, 'eval')() envs = SubprocVecEnv( [make_atari(C.env_id, r + 1, 'train') for r in range(4)]) model = Model(eval_env.observation_space.shape, eval_env.action_space.n) runner = Runner(envs, model.policy, nb_rollout=C.nb_rollout) sess.run(tf.global_variables_initializer()) writer = tf.summary.FileWriter( './{}/summary/{}'.format(C.env_id, time_stamp), sess.graph) for i in range(C.iterations): if i % C.eval_freq == 0: rewards = evaluate(eval_env, model.policy, C.eval_episodes) logger.log( f'Step: {i} | Max reward: {np.max(rewards)} | Mean reward: {np.mean(rewards):.2f} | Std: {np.std(rewards):.2f}'