def update_fx(): fx = request.args.get("code") current_app.settings['PORTFOLIO']['base_fx'] = fx update_config() from utils import fxsymbol as fxs current_app.fx = fxs(fx, 'all') regenerate_nav() redir = request.args.get("redirect") return redirect(redir)
def settings(): highlight = request.args.get('highlight', 'subtree') nouns = {} nouns['nsubj'] = request.args.get('nsubj', '0') nouns['nsubjpass'] = request.args.get('nsubjpass', '0') nouns['pobj'] = request.args.get('pobj', '0') nouns['dobj'] = request.args.get('dobj', '0') for key, value in nouns.items(): nouns[key] = True if value == '1' else False utils.update_config(configs, (highlight=='subtree'), nouns) return redirect(request.referrer)
def button_pushed(): """Actions button pushes by number """ if config.PUSHES == 1: if config.FIAT == 0: display.update_nocoin_screen() time.sleep(3) display.update_startup_screen() else: display.update_qr_request(config.SATS) config.INVOICE = qr.scan() while config.INVOICE is False: display.update_qr_failed() time.sleep(1) display.update_qr_request(config.SATS) config.INVOICE = qr.scan() update_payout_screen() if config.PUSHES == 2: if config.FIAT == 0: display.update_nocoin_screen() time.sleep(3) display.update_startup_screen() else: lntxbot.process_using_lnurl(config.SATS) if config.PUSHES == 3: display.update_lntxbot_scan() lntxcreds = lntxbot.scan_creds() utils.update_config("LNTXBOTCRED", lntxcreds) importlib.reload(config) balance = lntxbot.get_lnurl_balance() display.update_lntxbot_balance(balance) GPIO.cleanup() os.execv("/home/pi/LightningATM/app.py", [""]) if config.PUSHES == 4: logger.info("Button pushed three times (add coin)") print("Button pushed three times (add coin)") config.PULSES = 2 if config.PUSHES == 5: logger.warning("Button pushed three times (restart)") print("Button pushed three times (restart)") GPIO.cleanup() os.execv("/home/pi/LightningATM/app.py", [""]) if config.PUSHES == 6: display.update_shutdown_screen() GPIO.cleanup() logger.info("ATM shutdown (5 times button)") os.system("sudo shutdown -h now") config.PUSHES = 0
def parse_args(): parser = argparse.ArgumentParser(description='MiSLAS training (Stage-1)') parser.add_argument('--cfg', help='experiment configure file name', required=True, type=str) parser.add_argument('opts', help="Modify config options using the command-line", default=None, nargs=argparse.REMAINDER) args = parser.parse_args() update_config(config, args) return args
def __init__(self): os.environ["TCL_LIBRARY"] = "/lib64/tcl8.5/" self.win = tk.Tk() self.win.title("Continuous Aggregate Monitor: Alarm Handler") self.win.configure(background=u.lightgrey_color) self.get_alarm_handler_style() self.delim = ',' self.pdelim = '=' parser = ArgumentParser() parser.add_argument("-f", "--file", dest="filename", help="Configuration File Location", metavar="FILE", default="alarmConfig.txt") args = vars(parser.parse_args()) # File array is literally an array of data with methods for reading a writing to disk self.conf = alarm_object.FILE_ARRAY(args['filename'],self.pdelim) # This method gets config data from the array u.parse_config(self.conf) # This update changes the config data for this instance of AlarmHandler (probably should be more like self.update_config(), but this works) u.update_config(self) # This is the initial get of alarm handler data from disk self.fileArray = alarm_object.FILE_ARRAY(self.filename,self.delim) if len(self.fileArray.filearray) == 0: print("ERROR: Null alarm input file, please resolve in configure file") self.quit() # This is the initial get of alarm handler's previous instance history data from disk self.HL = alarm_object.HISTORY_LIST(self.histfilename,self.delim,self.pdelim,self.timeWait) # This tacks on to the end of the alarm handler data and "external" alarm information - allows an online analyzer or standalone script to supplement alarms into this GUI if os.path.exists(self.externalFilename): # Special case for running in an external situation like Japan or camguin analysis self.externalFileArray = alarm_object.FILE_ARRAY(self.externalFilename,self.delim) else: self.externalFileArray = None # FIXME here is the instantiation into memory of alarm data self.OL = alarm_object.OBJECT_LIST(self.fileArray,self.cooldownLength) # Alarm indicator image, also serves as a sound checker and GUI refresh when clicked self.masterAlarmImage = tk.PhotoImage(file='ok.ppm').subsample(2) self.masterAlarmButton = tk.Label(self.win, image=self.masterAlarmImage, cursor="hand2", bg=u.lightgrey_color) # This is the TCP/IP connection to the alarm sound server self.alarmClient = bclient.sockClient(self.remoteName) # Loop checks alarms self.alarmLoop = alarm_object.ALARM_LOOP(self) # Loop controls GUI refreshes self.alarmLoopGUI = alarm_object.ALARM_LOOP_GUI(self) # Loop controls sound making in the background self.alarmLoopMonitor = alarm_object.ALARM_LOOP_MONITOR(self) # Creates all GUI tabs self.tabs = self.create_widgets()
async def save_params(req, resp): await picoweb.start_response(resp, content_type='application/json', headers={"Access-Control-Allow-Origin": "*"}) size = int(req.headers[b"Content-Length"]) data = await req.reader.readexactly(size) data_txt = data.decode('utf-8') params = json.loads(data_txt) atmp = params["aliases"] aliases = {} for al in atmp.split("\n"): if al != '': key = al.split("=")[0] val = al.split("=")[1] aliases[key] = val print(aliases) params["aliases"] = aliases print(params) utils.update_config(params) res = {"result": "Dane zapisane"} await resp.awrite(json.dumps(res).encode('utf-8')) await resp.drain()
def tor_services(): action = request.args.get("action") if action == 'start': current_app.settings['SERVER']['onion_server'] = 'True' update_config() from stem.control import Controller from urllib.parse import urlparse current_app.tor_port = current_app.settings['SERVER'].getint('onion_port') current_app.port = current_app.settings['SERVER'].getint('port') from warden_modules import home_path toraddr_file = os.path.join(home_path(), "onion.txt") current_app.save_tor_address_to = toraddr_file proxy_url = "socks5h://localhost:9050" tor_control_port = "" try: tor_control_address = urlparse(proxy_url).netloc.split(":")[0] if tor_control_address == "localhost": tor_control_address = "127.0.0.1" current_app.controller = Controller.from_port( address=tor_control_address, port=int(tor_control_port) if tor_control_port else "default", ) except Exception: current_app.controller = None from tor import start_hidden_service start_hidden_service(current_app) flash(f"Started Tor Hidden Services at {current_app.tor_service_id}.onion", "success") if action == 'stop': current_app.settings['SERVER']['onion_server'] = 'False' update_config() from tor import stop_hidden_services stop_hidden_services(current_app) flash("Stopped Tor Hidden Services", "warning") return render_template("warden/tor.html", title="Tor Hidden Services", current_app=current_app)
def eval_roads(): global config rows, cols = 1344, 1344 config = update_config(config, target_rows=rows, target_cols=cols) ds = ReadingImageProvider(RawImageTypePad, paths, fn_mapping, image_suffix=image_suffix) folds = [([], list(range(len(ds)))) for i in range(4)] num_workers = 0 if os.name == 'nt' else 2 keval = FullImageEvaluator(config, ds, test=test, flips=3, num_workers=num_workers, border=22) for fold, (t, e) in enumerate(folds): if args.fold is not None and int(args.fold) != fold: continue keval.predict(fold, e)
def eval_roads(): global config rows, cols = 512, 512 config = update_config(config, target_rows=rows, target_cols=cols) ds = ReadingImageProvider(RawImageTypePad, paths, fn_mapping, image_suffix=image_suffix) folds = [([], list(range(len(ds)))) for i in range(4)] num_workers = 0 if os.name == 'nt' else 2 keval = FullImageEvaluator(config, ds, test=test, flips=3, num_workers=num_workers, border=0) for fold, (t, e) in enumerate(folds): if args.fold is not None and int(args.fold) != fold: continue keval.predict(fold, e)
def check_dangermode(): """Checks if DANGERMODE is YES or NO """ if config.DANGERMODE == "NO": utils.update_config("LNTXBOTCRED", "") utils.update_config("LNDMACAROON", "") utils.update_config("ACTIVEWALLET", "") importlib.reload(config) elif config.DANGERMODE == "YES": pass else: logger.info("ATM shutdown (DANGERMODE isn't set properly)") GPIO.cleanup() os.system("sudo shutdown -h now")
def read_or_write_config(filepath): """ GET: 读取配置文件并在页面显示 POST: 更新配置文件, 返回依照页面上填写信息生成的shell命令 """ if request.method == 'GET': return jsonify(**get_config(filepath)) if request.method == 'POST': configs = {t[0]: t[1] for t in request.form.items()} configs["channel"] = request.form.getlist("channel") if "merge-json-local" not in configs: configs["merge-json-local"] = '' update_config('merge.xml', configs) update_config(configs['core-site'], configs) update_config(configs['mapred-site'], configs) update_config(configs['hbase-site'], configs) MERGE_JSON_DIR = json.load(open("../config.json"))["merge_json_dir"] merge_json_local = os.path.join(MERGE_JSON_DIR, configs['merge-json-local']) return_json = update_and_fetch_mrtask_script(configs) return_json.update({'merge_json_local': merge_json_local}) return jsonify(return_json)
def train(config): self_summary_strs = [] # summary string to print out for later # first, read both data and filter stuff, to get the word2vec idx, train_data = read_data(config, 'train', config.load) val_data = read_data( config, 'val', True ) # dev should always load model shared data(word2idx etc.) from train config_vars = vars(config) str_ = "threshold setting--\n" + "\t" + " ,".join( ["%s:%s" % (key, config_vars[key]) for key in config.thresmeta]) print str_ self_summary_strs.append(str_) # cap the numbers # max sentence word count etc. update_config(config, [train_data, val_data], showMeta=True) # all word num is <= max_thres str_ = "renewed ----\n" + "\t" + " ,".join( ["%s:%s" % (key, config_vars[key]) for key in config.maxmeta]) print str_ self_summary_strs.append(str_) # now we initialize the matrix for word embedding for word not in glove word2vec_dict = train_data.shared['word2vec'] word2idx_dict = train_data.shared[ 'word2idx'] # this is the word not in word2vec # we are not fine tuning , so this should be empty idx2vec_dict = { word2idx_dict[word]: vec for word, vec in word2vec_dict.items() if word in word2idx_dict } # random initial embedding matrix for new words config.emb_mat = np.array([ idx2vec_dict[idx] if idx2vec_dict.has_key(idx) else np.random.multivariate_normal( np.zeros(config.word_emb_size), np.eye(config.word_emb_size)) for idx in xrange(config.word_vocab_size) ], dtype="float32") model = get_model(config) # construct model under gpu0 trainer = Trainer(model, config) tester = Tester(model, config) saver = tf.train.Saver(max_to_keep=5) # how many model to keep bestsaver = tf.train.Saver(max_to_keep=5) # just for saving the best model save_period = config.save_period # also the eval period # for debug, show the batch content if (config.showspecs): for batch in train_data.get_batches(2, num_steps=20): batchIdx, batchDs = batch print "showing a batch with batch_size=2" # show each data point print "keys:%s" % batchDs.data.keys() for key in sorted(batchDs.data.keys()): print "\t%s:%s" % (key, batchDs.data[key]) # show some image feature photo_idx1 = batchDs.data['photo_idxs'][0][0][ 0] # [bacth_num][album_num][photo_num] photo_id1 = batchDs.data['photo_ids'][0][0][0] photo_idx2 = batchDs.data['photo_idxs'][1][0][0] photo_id2 = batchDs.data['photo_ids'][1][0][0] print "pidx:%s,pid:%s,feature:\n %s (%s)\n,should be:\n %s (%s)" % ( photo_idx1, photo_id1, batchDs.data['pidx2feat'][photo_idx1][:10], batchDs.data['pidx2feat'][photo_idx1].shape, train_data.shared['pid2feat'][photo_id1][:10], train_data.shared['pid2feat'][photo_id1].shape) print "pidx:%s,pid:%s,feature:\n %s (%s)\n,should be:\n %s (%s)" % ( photo_idx2, photo_id2, batchDs.data['pidx2feat'][photo_idx2][:10], batchDs.data['pidx2feat'][photo_idx2].shape, train_data.shared['pid2feat'][photo_id2][:10], train_data.shared['pid2feat'][photo_id2].shape) # get the feed_dict to check #feed_dict = model.get_feed_dict(batchDs,is_train=True) feed_dict = model.get_feed_dict(batchDs, is_train=False) sys.exit() # start training! # allow_soft_placement : tf will auto select other device if the tf.device(*) not available tfconfig = tf.ConfigProto(allow_soft_placement=True) tfconfig.gpu_options.allow_growth = True # this way it will only allocate nessasary gpu, not take all with tf.Session(config=tfconfig) as sess: # calculate total parameters totalParam = cal_total_param() str_ = "total parameters: %s" % (totalParam) print str_ self_summary_strs.append(str_) initialize(load=config.load, load_best=config.load_best, model=model, config=config, sess=sess) # the total step (iteration) the model will run last_time = time.time() # total / batchSize * epoch num_steps = int( math.ceil(train_data.num_examples / float(config.batch_size))) * config.num_epochs # get_batches is a generator, run on the fly # there will be num_steps batch str_ = " batch_size:%s, epoch:%s,total step:%s,eval/save every %s steps" % ( config.batch_size, config.num_epochs, num_steps, config.save_period) print str_ self_summary_strs.append(str_) best = { "acc": 0.0, "step": -1 } # remember the best eval acc during training finalAcc = None isStart = True for batch in tqdm(train_data.get_batches(config.batch_size, num_steps=num_steps), total=num_steps): # each batch has (batch_idxs,Dataset(batch_data, full_shared)) # batch_data has {"q":,"y":..."pidx2feat",.."photo_idxs"..} global_step = sess.run(model.global_step) + 1 # start from 0 # if load from existing model, save if first if config.load and isStart: tqdm.write("saving original model...") tqdm.write("\tsaving model...") saver.save(sess, config.save_dir_model, global_step=global_step) tqdm.write("\tdone") isStart = False id2predanswers = {} id2realanswers = {} for evalbatch in val_data.get_batches( config.batch_size, num_steps=config.val_num_batches, shuffle=False, cap=True): yp = tester.step( sess, evalbatch ) # [N,4] # id2realanswersprob for each answer pred, gt = getAnswers( yp, evalbatch) # from here we get the qid:yindx, id2predanswers.update(pred) id2realanswers.update(gt) evalAcc = getEvalScore(id2predanswers, id2realanswers) tqdm.write( "\teval on validation %s batches Acc:%s, (best:%s at step %s) " % (config.val_num_batches, evalAcc, best['acc'], best['step'])) # remember the best acc if (evalAcc > best['acc']): best['acc'] = evalAcc best['step'] = global_step # save the best model tqdm.write("\t saving best model...") bestsaver.save(sess, config.save_dir_best_model, global_step=global_step) tqdm.write("\t done.") finalAcc = evalAcc loss, summary, train_op = trainer.step(sess, batch, get_summary=False) if global_step % save_period == 0: # time to save model duration = time.time() - last_time # in seconds sec_per_step = duration / float(save_period) last_time = time.time() #use tqdm to print tqdm.write( "step:%s/%s (epoch:%.3f), took %s, loss:%s, estimate remaining:%s" % (global_step, num_steps, (config.num_epochs * global_step / float(num_steps)), sec2time(duration), loss, sec2time((num_steps - global_step) * sec_per_step))) tqdm.write("\tsaving model...") saver.save(sess, config.save_dir_model, global_step=global_step) tqdm.write("\tdone") id2predanswers = {} id2realanswers = {} for evalbatch in val_data.get_batches( config.batch_size, num_steps=config.val_num_batches, shuffle=False, cap=True): yp = tester.step( sess, evalbatch ) # [N,4] # id2realanswersprob for each answer pred, gt = getAnswers( yp, evalbatch) # from here we get the qid:yindx, id2predanswers.update(pred) id2realanswers.update(gt) evalAcc = getEvalScore(id2predanswers, id2realanswers) tqdm.write( "\teval on validation %s batches Acc:%s, (best:%s at step %s) " % (config.val_num_batches, evalAcc, best['acc'], best['step'])) # remember the best acc if (evalAcc > best['acc']): best['acc'] = evalAcc best['step'] = global_step # save the best model tqdm.write("\t saving best model...") bestsaver.save(sess, config.save_dir_best_model, global_step=global_step) tqdm.write("\t done.") finalAcc = evalAcc if global_step % save_period != 0: # time to save model saver.save(sess, config.save_dir_model, global_step=global_step) str_ = "best eval on val Accurucy: %s at %s step, final step %s Acc is %s" % ( best['acc'], best['step'], global_step, finalAcc) print str_ self_summary_strs.append(str_) if config.write_self_sum: f = open(config.self_summary_path, "w") f.writelines("%s" % ("\n".join(self_summary_strs))) f.close()
def test(config): if config.is_test_on_val: test_data = read_data(config, 'val', True) print "total val samples:%s" % test_data.num_examples else: test_data = read_data( config, 'test', True ) # here will load shared.p from config.outpath (outbase/modelname/runId/) print "total test samples:%s" % test_data.num_examples # get the max_sent_size and other stuff print "threshold setting--" config_vars = vars(config) print "\t" + " ,".join( ["%s:%s" % (key, config_vars[key]) for key in config.thresmeta]) # cap the numbers update_config(config, [test_data], showMeta=True) print "renewed ----" print "\t" + " ,".join( ["%s:%s" % (key, config_vars[key]) for key in config.maxmeta]) model = get_model(config) # update each batch forward into this dict id2predanswers = {} id2realanswers = {} id2yp = {} tfconfig = tf.ConfigProto(allow_soft_placement=True) tfconfig.gpu_options.allow_growth = True # this way it will only allocate nessasary gpu, not take all # or you can set hard limit #tfconfig.gpu_options.per_process_gpu_memory_fraction = 0.4 with tf.Session(config=tfconfig) as sess: initialize(load=True, load_best=config.load_best, model=model, config=config, sess=sess) if config.is_save_weights: weight_dict = {} weight_sum = open(os.path.join(config.weights_path, "all.txt"), "w") for var in tf.trainable_variables(): shape = var.get_shape() weight_sum.writelines("%s %s\n" % (var.name, shape)) var_val = sess.run(var) weight_dict[var.name] = var_val np.savez(os.path.join(config.weights_path, "weights.npz"), **weight_dict) weight_sum.close() last_time = time.time() # num_epoch should be 1 num_steps = int( math.ceil(test_data.num_examples / float(config.batch_size))) * config.num_epochs # load the graph and variables tester = Tester(model, config, sess) count = 0 print "total step:%s" % num_steps for batch in tqdm(test_data.get_batches(config.batch_size, num_steps=num_steps, shuffle=False), total=num_steps): count += 1 yp = tester.step( sess, batch) # [N,4] # id2realanswersprob for each answer if config.get_yp: pred, gt, yp = getAnswers_yp(yp, batch) id2yp.update(yp) else: pred, gt = getAnswers(yp, batch) # from here we get the qid:yindx, id2predanswers.update(pred) id2realanswers.update(gt) acc = getEvalScore(id2predanswers, id2realanswers) print "done, got %s answers, accuracy:%s" % (len(id2predanswers), acc) json.dump(id2predanswers, open("%s/answers.json" % config.val_path, "w")) if config.get_yp: json.dump({id_: "%s" % (id2yp[id_]) for id_ in id2yp}, open("%s/yps.json" % config.val_path, "w"))
def train(config): self_summary_strs = [] # summary string to print out for later # first, read both data and filter stuff, to get the word2vec idx, train_data = read_data(config, 'train', config.load) val_data = read_data( config, 'val', True ) # dev should always load model shared data(word2idx etc.) from train # now that the dataset is loaded , we get the max_word_size from the dataset # then adjust the max based on the threshold as well # also get the vocab size config_vars = vars(config) str_ = "threshold setting--\n" + "\t" + " ,".join( ["%s:%s" % (key, config_vars[key]) for key in config.thresmeta]) print str_ self_summary_strs.append(str_) # cap the numbers # max sentence word count etc. update_config(config, [train_data, val_data], showMeta=True) # all word num is <= max_thres str_ = "renewed ----\n" + "\t" + " ,".join( ["%s:%s" % (key, config_vars[key]) for key in config.maxmeta]) print str_ self_summary_strs.append(str_) # now we initialize the matrix for word embedding for word not in glove word2vec_dict = train_data.shared['word2vec'] word2idx_dict = train_data.shared[ 'word2idx'] # this is the word not in word2vec # we are not fine tuning , so this should be empty idx2vec_dict = { word2idx_dict[word]: vec for word, vec in word2vec_dict.items() if word in word2idx_dict } #print len(idx2vec_dict) # 0 # config.word_vocab_size = len(train_data.shared['word2idx']) # the word not in word2vec # so the emb_mat should all be a random vector # np.random.multivariate_normal gets mean of zero and co of 1 for each dim, like #>>> np.random.multivariate_normal(np.zeros(5),np.eye(5)) #array([-0.73663652, -1.16417783, -0.74083293, -0.80016731, 0.060182 ]) # random initial embedding matrix for new words config.emb_mat = np.array([ idx2vec_dict[idx] if idx2vec_dict.has_key(idx) else np.random.multivariate_normal( np.zeros(config.word_emb_size), np.eye(config.word_emb_size)) for idx in xrange(config.word_vocab_size) ], dtype="float32") model = get_model(config) # construct model under gpu0 trainer = Trainer(model, config) tester = Tester(model, config) saver = tf.train.Saver(max_to_keep=5) # how many model to keep bestsaver = tf.train.Saver(max_to_keep=5) # just for saving the best model save_period = config.save_period # also the eval period # start training! # allow_soft_placement : tf will auto select other device if the tf.device(*) not available tfconfig = tf.ConfigProto(allow_soft_placement=True) tfconfig.gpu_options.allow_growth = True # this way it will only allocate nessasary gpu, not take all # or you can set hard limit #tfconfig.gpu_options.per_process_gpu_memory_fraction = 0.4 with tf.Session(config=tfconfig) as sess: # calculate total parameters totalParam = cal_total_param() str_ = "total parameters: %s" % (totalParam) print str_ self_summary_strs.append(str_) initialize(load=config.load, load_best=config.load_best, model=model, config=config, sess=sess) # the total step (iteration) the model will run last_time = time.time() # total / batchSize * epoch num_steps = int( math.ceil(train_data.num_examples / float(config.batch_size))) * config.num_epochs # get_batches is a generator, run on the fly # there will be num_steps batch str_ = " batch_size:%s, epoch:%s,total step:%s,eval/save every %s steps" % ( config.batch_size, config.num_epochs, num_steps, config.save_period) print str_ self_summary_strs.append(str_) best = { "acc": 0.0, "step": -1 } # remember the best eval acc during training finalAcc = None isStart = True for batch in tqdm(train_data.get_batches(config.batch_size, num_steps=num_steps), total=num_steps): # each batch has (batch_idxs,Dataset(batch_data, full_shared)) # batch_data has {"q":,"y":..."pidx2feat",.."photo_idxs"..} global_step = sess.run(model.global_step) + 1 # start from 0 # if load from existing model, save if first if config.load and isStart: tqdm.write("saving original model...") tqdm.write("\tsaving model...") saver.save(sess, config.save_dir_model, global_step=global_step) tqdm.write("\tdone") isStart = False id2predanswers = {} id2realanswers = {} for evalbatch in val_data.get_batches( config.batch_size, num_steps=config.val_num_batches, shuffle=False, cap=True): yp = tester.step( sess, evalbatch ) # [N,4] # id2realanswersprob for each answer pred, gt = getAnswers( yp, evalbatch) # from here we get the qid:yindx, id2predanswers.update(pred) id2realanswers.update(gt) evalAcc = getEvalScore(id2predanswers, id2realanswers) tqdm.write( "\teval on validation %s batches Acc:%s, (best:%s at step %s) " % (config.val_num_batches, evalAcc, best['acc'], best['step'])) # remember the best acc if (evalAcc > best['acc']): best['acc'] = evalAcc best['step'] = global_step # save the best model tqdm.write("\t saving best model...") bestsaver.save(sess, config.save_dir_best_model, global_step=global_step) tqdm.write("\t done.") finalAcc = evalAcc loss, train_op = trainer.step(sess, batch) if global_step % save_period == 0: # time to save model duration = time.time() - last_time # in seconds sec_per_step = duration / float(save_period) last_time = time.time() #use tqdm to print tqdm.write( "step:%s/%s (epoch:%.3f), took %s, loss:%s, estimate remaining:%s" % (global_step, num_steps, (config.num_epochs * global_step / float(num_steps)), sec2time(duration), loss, sec2time((num_steps - global_step) * sec_per_step))) tqdm.write("\tsaving model...") saver.save(sess, config.save_dir_model, global_step=global_step) tqdm.write("\tdone") id2predanswers = {} id2realanswers = {} for evalbatch in val_data.get_batches( config.batch_size, num_steps=config.val_num_batches, shuffle=False, cap=True): yp = tester.step( sess, evalbatch ) # [N,4] # id2realanswersprob for each answer pred, gt = getAnswers( yp, evalbatch) # from here we get the qid:yindx, id2predanswers.update(pred) id2realanswers.update(gt) evalAcc = getEvalScore(id2predanswers, id2realanswers) tqdm.write( "\teval on validation %s batches Acc:%s, (best:%s at step %s) " % (config.val_num_batches, evalAcc, best['acc'], best['step'])) # remember the best acc if (evalAcc > best['acc']): best['acc'] = evalAcc best['step'] = global_step # save the best model tqdm.write("\t saving best model...") bestsaver.save(sess, config.save_dir_best_model, global_step=global_step) tqdm.write("\t done.") finalAcc = evalAcc if global_step % save_period != 0: # time to save model saver.save(sess, config.save_dir_model, global_step=global_step) str_ = "best eval on val Accurucy: %s at %s step, final step %s Acc is %s" % ( best['acc'], best['step'], global_step, finalAcc) print str_ self_summary_strs.append(str_) if config.write_self_sum: f = open(config.self_summary_path, "w") f.writelines("%s" % ("\n".join(self_summary_strs))) f.close()
def specter_auth(): if request.method == 'GET': templateData = { "title": "Login to Specter", "current_app": current_app, "current_user": current_user } return (render_template('warden/specter_auth.html', **templateData)) if request.method == 'POST': from message_handler import Message current_app.message_handler.clean_category('Specter Connection') url = request.form.get('url') url = url_parser(url) # Try to ping this url if 'onion' not in url: try: status_code = requests.head(url).status_code except Exception as e: flash(f'Please check Specter URL. Error: {e}', 'danger') else: try: status_code = tor_request(url).status_code except Exception as e: flash(f'Please check Specter URL. Error: {e}', 'danger') try: if int(status_code) < 400: message = Message( category='Specter Connection', message_txt='Pinging URL', notes= f"{url}<br> ping <span class='text-success'>✅ Success</span>" ) current_app.message_handler.add_message(message) else: flash('Please check Specter URL (unreacheable)', 'danger') return redirect(url_for('warden.specter_auth')) except Exception as e: flash(f'Error Connecting. Error: {e}', 'danger') return redirect(url_for('warden.specter_auth')) # Try to authenticate try: current_app.specter.base_url = url current_app.specter.login_url = url + 'auth/login' current_app.specter.tx_url = url + 'wallets/wallets_overview/txlist' current_app.specter.core_url = url + 'settings/bitcoin_core?' current_app.specter.login_payload = { 'username': request.form.get('username'), 'password': request.form.get('password') } session = current_app.specter.init_session() session.close() except Exception as e: flash(f'Error logging in to Specter: {e}', 'danger') return redirect(url_for('warden.specter_auth')) current_app.downloading = True current_app.settings['SPECTER']['specter_url'] = url current_app.settings['SPECTER']['specter_login'] = request.form.get( 'username') current_app.settings['SPECTER']['specter_password'] = request.form.get( 'password') update_config() current_app.specter = Specter() current_app.specter.refresh_txs(load=False) flash("Success. Connected to Specter Server.", "success") # Now allow download of all txs in background on next run return redirect(url_for('warden.warden_page'))
def test(config): if config.is_test_on_val: test_data = read_data(config, 'val', True, subset=False) print "total val samples:%s" % test_data.num_examples else: test_data = read_data( config, 'test', True, subset=False ) # here will load shared.p from config.outpath (outbase/modelname/runId/) print "total test samples:%s" % test_data.num_examples config.imgfeat_dim = test_data.imgfeat_dim # get the max_sent_size and other stuff print "threshold setting--" config_vars = vars(config) print "\t" + " ,".join( ["%s:%s" % (key, config_vars[key]) for key in config.thresmeta]) # cap the numbers update_config(config, [test_data], showMeta=True) print "renewed ----" print "\t" + " ,".join( ["%s:%s" % (key, config_vars[key]) for key in config.maxmeta]) model = get_model(config) tfconfig = tf.ConfigProto(allow_soft_placement=True) tfconfig.gpu_options.allow_growth = True # this way it will only allocate nessasary gpu, not take all # or you can set hard limit #tfconfig.gpu_options.per_process_gpu_memory_fraction = 0.4 with tf.Session(config=tfconfig) as sess: initialize(load=True, load_best=config.load_best, model=model, config=config, sess=sess) if config.is_pack_model: saver = tf.train.Saver() global_step = model.global_step # put input and output to a universal name for reference when in deployment # find the nessary stuff in model.get_feed_dict # multiple input tf.add_to_collection("sents", model.sents) tf.add_to_collection("sents_c", model.sents_c) tf.add_to_collection("sents_mask", model.sents_mask) tf.add_to_collection("pis", model.pis) # image and text feature tf.add_to_collection("image_emb_mat", model.image_emb_mat) tf.add_to_collection("existing_emb_mat", model.existing_emb_mat) # for getting the highest ranked photo #tf.add_to_collection("att_logits",model.att_logits) tf.add_to_collection( "is_train", model.is_train) # TODO, change this to a constant tf.add_to_collection("output", model.s) # also save all the model config and note into the model pack_model_note = tf.get_variable( "model_note", shape=[], dtype=tf.string, initializer=tf.constant_initializer(config.pack_model_note), trainable=False) full_config = tf.get_variable("model_config", shape=[], dtype=tf.string, initializer=tf.constant_initializer( json.dumps(vars(config))), trainable=False) print "saving packed model" # the following wont save the var model_note, model_config that's not in the graph, # TODO: fix this """ # put into one big file to save input_graph_def = tf.get_default_graph().as_graph_def() #print [n.name for n in input_graph_def.node] # We use a built-in TF helper to export variables to constants output_graph_def = tf.graph_util.convert_variables_to_constants( sess, # The session is used to retrieve the weights input_graph_def, # The graph_def is used to retrieve the nodes [tf.get_collection("output")[0].name.split(":")[0]] # The output node names are used to select the usefull nodes ) output_graph = os.path.join(config.pack_model_path,"final.pb") # Finally we serialize and dump the output graph to the filesystem with tf.gfile.GFile(output_graph, "wb") as f: f.write(output_graph_def.SerializeToString()) print("%d ops in the final graph." % len(output_graph_def.node)) """ # save it into a path with multiple files saver.save(sess, os.path.join(config.pack_model_path, "final"), global_step=global_step) print "model saved in %s" % (config.pack_model_path) return if config.is_save_weights: weight_dict = {} weight_sum = open(os.path.join(config.weights_path, "all.txt"), "w") for var in tf.trainable_variables(): shape = var.get_shape() weight_sum.writelines("%s %s\n" % (var.name, shape)) var_val = sess.run(var) weight_dict[var.name] = var_val np.savez(os.path.join(config.weights_path, "weights.npz"), **weight_dict) weight_sum.close() last_time = time.time() # num_epoch should be 1 num_steps = int( math.ceil(test_data.num_examples / float(config.batch_size))) * config.num_epochs # load the graph and variables tester = Tester(model, config, sess) perf = evaluate(test_data, config, sess, tester) print "performance:" print perf metric = ["r@1", "r@5", "r@10", "mr"] metrics = [] for one in metric: metrics.append("%s_%s" % ("t2i", one)) metrics.append("%s_%s" % ("i2t", one)) print "\t".join([m for m in metrics]) print "\t".join(["%.4f" % perf[m] for m in metrics]) if config.save_answers: pickle.dump(imgid2sents, open("%s/answers_i2t.p" % config.val_path, "w")) pickle.dump(sentId2imgs, open("%s/answers_t2i.p" % config.val_path, "w"))
folds = [([], list(range(len(ds)))) for i in range(5)] num_workers = 0 if os.name == 'nt' else 4 keval = GdalFullEvaluator(config, ds, folds, test=True, flips=3, num_workers=num_workers, border=0, crowdai=args.dataset.lower()=='crowdai') keval.predict() if __name__ == "__main__": if args.dataset.lower() == 'spacenet': print("Spacenet is true") print(os.path.join(config.results_dir, 'results', config.folder, 'merged'), test_data_path, out_file) print(config.num_channels) in_shape = 2048 test_shape = 2048 if args.dataset.lower() == 'spacenet': in_shape = 650 test_shape = 512 elif args.dataset.lower() == 'crowdai': in_shape = 300 test_shape = 300 config = update_config(config, img_rows=in_shape, img_cols=in_shape, target_rows=test_shape, target_cols=test_shape, num_channels=config.num_channels) print("predicting stage 1/3") predict() print("predicting stage 2/3") merge_tiffs(os.path.join(config.results_dir, 'results', config.folder)) print("predicting stage 3/3") make_submission(os.path.join(config.results_dir, 'results', config.folder, 'merged'), test_data_path, out_file, spacenet=args.dataset.lower())
im_type = SalImageType if three: im_type = TiffImageType ds = ReadingImageProvider(im_type, paths, fn_mapping, image_suffix='RGB') folds = get_folds(ds, 5) num_workers = 0 if os.name == 'nt' else 8 train(ds, folds, config, num_workers=num_workers, transforms=augment_flips_color) if __name__ == "__main__": three = (config.num_channels == 3) num_epochs = config.nb_epoch config = update_config(config, num_channels=3, nb_epoch=5) print("start training stage 1/3") train_stage0() print("start training stage 2/3") config = update_config(config, num_channels=config.num_channels, nb_epoch=num_epochs) train_stage1(config.sal_map, three=three) print("start training stage 3/3") config = update_config(config, loss=config.loss + '_w', nb_epoch=num_epochs + 2) train_stage2(config.sal_map, three=three)
def refresh_screen(self,alarmHandlerWindow): u.update_config(alarmHandlerWindow) self.update_settings(alarmHandlerWindow)
print "=" * 60, "\nChecking Dependencies and Configure\n", "=" * 60 print "Tempdir is:", tempdir mainLogger.info("Checking all Dependencies...") mainLogger.debug("truerypt binary is {}".format(parser.get("truecrypting", "tc_binary"))) dependency_check([parser.get("truecrypting", "tc_binary"), "--text", "--version"]) mainLogger.info("Configuring...") # # Setting Parameters given from argparse # update_config("DEFAULT", "device", args.device) update_config("thunderbird_linux", "version", args.thunder) update_config("thunderbird_windows", "version", args.thunder) update_config("thunderbird_mac", "version", args.thunder) update_config("vidalia_linux", "version", args.vidalia) update_config("vidalia_windows", "version", args.vidalia) update_config("vidalia_mac", "version", args.vidalia) update_config("DEFAULT", "container_name", args.container_name) update_config("truecrypting", "size", args.container_size) print "=" * 60, "\nMounting and Truecrypting\n", "=" * 60 # # Use an USB-Stick given by a Parameter or a detected one: #
def test(config): if config.is_test_on_val: test_data = read_data(config, 'val', True) print "total val samples:%s" % test_data.num_examples else: test_data = read_data( config, 'test', True ) # here will load shared.p from config.outpath (outbase/modelname/runId/) print "total test samples:%s" % test_data.num_examples # get the max_sent_size and other stuff print "threshold setting--" config_vars = vars(config) print "\t" + " ,".join( ["%s:%s" % (key, config_vars[key]) for key in config.thresmeta]) # cap the numbers update_config(config, [test_data], showMeta=True) # a hack for dmn model, since we fix the number of album, we need to keep the same during test if config.dmnplus: config.max_num_albums = config.num_albums_thres print "renewed ----" print "\t" + " ,".join( ["%s:%s" % (key, config_vars[key]) for key in config.maxmeta]) model = get_model(config) # update each batch forward into this dict id2predanswers = {} id2realanswers = {} id2yp = {} tfconfig = tf.ConfigProto(allow_soft_placement=True) tfconfig.gpu_options.allow_growth = True # this way it will only allocate nessasary gpu, not take all # or you can set hard limit with tf.Session(config=tfconfig) as sess: initialize(load=True, load_best=config.load_best, model=model, config=config, sess=sess) if config.is_pack_model: saver = tf.train.Saver() global_step = model.global_step # put input and output to a universal name for reference when in deployment # find the nessary stuff in model.get_feed_dict # multiple input tf.add_to_collection("at", model.at) tf.add_to_collection("at_c", model.at_c) tf.add_to_collection("at_mask", model.at_mask) tf.add_to_collection("ad", model.ad) tf.add_to_collection("ad_c", model.ad_c) tf.add_to_collection("ad_mask", model.ad_mask) tf.add_to_collection("when", model.when) tf.add_to_collection("when_c", model.when_c) tf.add_to_collection("when_mask", model.when_mask) tf.add_to_collection("where", model.where) tf.add_to_collection("where_c", model.where_c) tf.add_to_collection("where_mask", model.where_mask) tf.add_to_collection("pts", model.pts) tf.add_to_collection("pts_c", model.pts_c) tf.add_to_collection("pts_mask", model.pts_mask) tf.add_to_collection("pis", model.pis) tf.add_to_collection("pis_mask", model.pis_mask) tf.add_to_collection("q", model.q) tf.add_to_collection("q_c", model.q_c) tf.add_to_collection("q_mask", model.q_mask) tf.add_to_collection("choices", model.choices) tf.add_to_collection("choices_c", model.choices_c) tf.add_to_collection("choices_mask", model.choices_mask) # image and text feature tf.add_to_collection("image_emb_mat", model.image_emb_mat) tf.add_to_collection("existing_emb_mat", model.existing_emb_mat) # for getting the highest ranked photo tf.add_to_collection("att_logits", model.att_logits) tf.add_to_collection( "is_train", model.is_train) # TODO, change this to a constant tf.add_to_collection("output", model.yp) # also save all the model config and note into the model pack_model_note = tf.get_variable( "model_note", shape=[], dtype=tf.string, initializer=tf.constant_initializer(config.pack_model_note), trainable=False) full_config = tf.get_variable("model_config", shape=[], dtype=tf.string, initializer=tf.constant_initializer( json.dumps(vars(config))), trainable=False) print "saving packed model" # the following wont save the var model_note, model_config that's not in the graph, # TODO: fix this """ # put into one big file to save input_graph_def = tf.get_default_graph().as_graph_def() #print [n.name for n in input_graph_def.node] # We use a built-in TF helper to export variables to constants output_graph_def = tf.graph_util.convert_variables_to_constants( sess, # The session is used to retrieve the weights input_graph_def, # The graph_def is used to retrieve the nodes [tf.get_collection("output")[0].name.split(":")[0]] # The output node names are used to select the usefull nodes ) output_graph = os.path.join(config.pack_model_path,"final.pb") # Finally we serialize and dump the output graph to the filesystem with tf.gfile.GFile(output_graph, "wb") as f: f.write(output_graph_def.SerializeToString()) print("%d ops in the final graph." % len(output_graph_def.node)) """ # save it into a path with multiple files saver.save(sess, os.path.join(config.pack_model_path, "final"), global_step=global_step) print "model saved in %s" % (config.pack_model_path) return if config.is_save_weights: weight_dict = {} weight_sum = open(os.path.join(config.weights_path, "all.txt"), "w") for var in tf.trainable_variables(): shape = var.get_shape() weight_sum.writelines("%s %s\n" % (var.name, shape)) var_val = sess.run(var) weight_dict[var.name] = var_val np.savez(os.path.join(config.weights_path, "weights.npz"), **weight_dict) weight_sum.close() last_time = time.time() # num_epoch should be 1 num_steps = int( math.ceil(test_data.num_examples / float(config.batch_size))) * config.num_epochs # load the graph and variables tester = Tester(model, config, sess) count = 0 print "total step:%s" % num_steps for batch in tqdm(test_data.get_batches(config.batch_size, num_steps=num_steps, shuffle=False), total=num_steps): count += 1 if config.is_save_vis: # save all variables to pickle for visualization batchIdxs, batch_data = batch qid = batch_data.data['qid'] yp, C, C_win, att_logits, q_att_logits, at_mask, ad_mask, when_mask, where_mask, pts_mask, pis_mask, q_mask, hat_len, had_len, hwhen_len, hwhere_len, hpts_len, hpis_len, JXP, warp_h, h, at, ad, when, where, pts, pis, q = tester.step_vis( sess, batch) # each batch save as a pickle file # these all should have the same order vis = { "yp": yp, "data": batch_data.data, "C": C, "C_win": C_win, "att_logits": att_logits, "q_att_logits": q_att_logits, "at_mask": at_mask, "ad_mask": ad_mask, "when_mask": when_mask, "where_mask": where_mask, "pts_mask": pts_mask, "pis_mask": pis_mask, "q_mask": q_mask, "hat_len": hat_len, "had_len": had_len, "hwhen_len": hwhen_len, 'hwhere_len': hwhere_len, "hpts_len": hpts_len, "hpis_len": hpis_len, "photo_title_len": JXP, "warp_h": warp_h, "h": h, "at": at, "ad": ad, "when": when, "where": where, "pts": pts, "pis": pis, "q": q } """ print batch_data.data['qid'] print batch_data.data['q'] batch_data.data['cs'][0].insert(batch_data.data['yidx'][0],batch_data.data['y'][0]) print batch_data.data['cs'][0] print batch_data.data['photo_ids'] print yp print batch_data.data['pidx2feat'] print batch_data.data['photo_idxs'] sys.exit() """ pickle.dump( vis, open(os.path.join(config.vis_path, "%s.p" % count), "wb")) else: yp = tester.step( sess, batch) # [N,4] # id2realanswersprob for each answer if config.get_yp: pred, gt, yp = getAnswers_yp(yp, batch) id2yp.update(yp) else: pred, gt = getAnswers(yp, batch) # from here we get the qid:yindx, id2predanswers.update(pred) id2realanswers.update(gt) acc = getEvalScore(id2predanswers, id2realanswers) print "done, got %s answers, accuracy:%s" % (len(id2predanswers), acc) json.dump(id2predanswers, open("%s/answers.json" % config.val_path, "w")) if config.get_yp: json.dump({id_: "%s" % (id2yp[id_]) for id_ in id2yp}, open("%s/yps.json" % config.val_path, "w"))
try: val = utils.literal_eval(val) except ValueError, e: pass except SyntaxError, e: pass val = str(val) if utils.assertor.assert_nonempty_str(val): arg_to = utils.filer.validate_path_format(val) elif opt in ('-c', '--config'): utils.assertor.assert_type(val, dict) for key, value in val.items(): utils.update_config(key, value, 'command-line', silence=False) else: raise ValueError("Unknown option '%s'." % opt) # if len(unknowns) > 0: if unknowns: raise ValueError("Unknown option(s) %s." % unknowns) return arg_root, arg_from, arg_epoch, arg_name, arg_to except: raise @staticmethod
parser = argparse.ArgumentParser() parser.add_argument('config_path') parser.add_argument('--fold', type=int) parser.add_argument('--training', action='store_true') args = parser.parse_args() with open(args.config_path, 'r') as f: cfg = json.load(f) config = Config(**cfg) skip_folds = [] if args.fold is not None: skip_folds = [i for i in range(4) if i != int(args.fold)] test = not args.training config = update_config(config, dataset_path=os.path.join(config.dataset_path, 'test' if test else 'train')) paths = { 'masks': os.path.join(config.dataset_path, 'masks2m'), 'images': os.path.join(config.dataset_path, 'images') } fn_mapping = { 'masks': lambda name: os.path.splitext(name)[0] + '.png' } image_suffix = None def train_roads(): ds = ReadingImageProvider(RawImageType, paths, fn_mapping, image_suffix=image_suffix)
def train(config): self_summary_strs = [] # summary string to print out for later val_perf = [] # summary of validation performance # first, read both data and filter stuff, to get the word2vec idx, train_data = read_data(config, 'train', config.load) config.imgfeat_dim = train_data.imgfeat_dim val_data = read_data( config, 'val', True, subset=False ) # dev should always load model shared data(word2idx etc.) from train # now that the dataset is loaded , we get the max_word_size from the dataset # then adjust the max based on the threshold as well # also get the vocab size config_vars = vars(config) str_ = "threshold setting--\n" + "\t" + " ,".join( ["%s:%s" % (key, config_vars[key]) for key in config.thresmeta]) print str_ self_summary_strs.append(str_) # cap the numbers # max sentence word count etc. update_config(config, [train_data, val_data], showMeta=True) # all word num is <= max_thres str_ = "renewed ----\n" + "\t" + " ,".join( ["%s:%s" % (key, config_vars[key]) for key in config.maxmeta]) print str_ self_summary_strs.append(str_) # now we initialize the matrix for word embedding for word not in glove word2vec_dict = train_data.shared[ 'word2vec'] # empty if not use pre-train vector word2idx_dict = train_data.shared[ 'word2idx'] # this is the word not in word2vec # for finetuning or not using word2vec then it is all the word # we are not fine tuning , so this should be empty; empty if not use pretrain vector # if finetuning , this will have glove idx2vec_dict = { word2idx_dict[word]: vec for word, vec in word2vec_dict.items() if word in word2idx_dict } print "len(idx2vec_dict):%s,word_vocab_size:%s" % (len(idx2vec_dict), config.word_vocab_size) # config.word_vocab_size = len(train_data.shared['word2idx']) # the word not in word2vec # so the emb_mat should all be a random vector # np.random.multivariate_normal gets mean of zero and co of 1 for each dim, like #>>> np.random.multivariate_normal(np.zeros(5),np.eye(5)) #array([-0.73663652, -1.16417783, -0.74083293, -0.80016731, 0.060182 ]) # random initial embedding matrix for new words # this will take a long time when vocab_size is 6k if not config.no_wordvec: config.emb_mat = np.array([ idx2vec_dict[idx] if idx2vec_dict.has_key(idx) else np.random.multivariate_normal( np.zeros(config.word_emb_size), np.eye(config.word_emb_size)) for idx in xrange(config.word_vocab_size) ], dtype="float32") model = get_model(config) # construct model under gpu0 #for var in tf.trainable_variables(): # print var.name,var.get_shape() #sys.exit() trainer = Trainer(model, config) tester = Tester(model, config) saver = tf.train.Saver(max_to_keep=5) # how many model to keep bestsaver = tf.train.Saver(max_to_keep=5) # just for saving the best model save_period = config.save_period # also the eval period # start training! # allow_soft_placement : tf will auto select other device if the tf.device(*) not available tfconfig = tf.ConfigProto(allow_soft_placement=True) tfconfig.gpu_options.allow_growth = True # this way it will only allocate nessasary gpu, not take all # or you can set hard limit #tfconfig.gpu_options.per_process_gpu_memory_fraction = 0.4 with tf.Session(config=tfconfig) as sess: # calculate total parameters totalParam = cal_total_param() str_ = "total parameters: %s" % (totalParam) print str_ self_summary_strs.append(str_) initialize(load=config.load, load_best=config.load_best, model=model, config=config, sess=sess) # the total step (iteration) the model will run last_time = time.time() # total / batchSize * epoch num_steps = int( math.ceil(train_data.num_examples / float(config.batch_size))) * config.num_epochs # get_batches is a generator, run on the fly # there will be num_steps batch str_ = " batch_size:%s, epoch:%s, %s step every epoch, total step:%s,eval/save every %s steps" % ( config.batch_size, config.num_epochs, math.ceil(train_data.num_examples / float(config.batch_size)), num_steps, config.save_period) print str_ self_summary_strs.append(str_) metric = "t2i_r@5" # TODO: change this? best = { metric: 0.0, "step": -1 } # remember the best eval acc during training finalperf = None isStart = True for batch in tqdm(train_data.get_batches(config.batch_size, no_img_feat=True, num_steps=num_steps), total=num_steps, ascii=True): # each batch has (batch_idxs,Dataset(batch_data, full_shared)) # batch_data has {"q":,"y":..."pidx2feat",.."photo_idxs"..} global_step = sess.run(model.global_step) + 1 # start from 0 # if load from existing model, save if first if (global_step % save_period == 0) or (config.load_best and isStart) or (config.load and isStart and (config.ignore_vars is None)): isStart = False tqdm.write("\tsaving model %s..." % global_step) saver.save(sess, config.save_dir_model, global_step=global_step) tqdm.write("\tdone") evalperf = evaluate(val_data, config, sess, tester) tqdm.write( "\teval on validation:%s, (best %s:%s at step %s) " % (evalperf, metric, best[metric], best['step'])) # remember the best acc if (evalperf[metric] > best[metric]): best[metric] = evalperf[metric] best['step'] = global_step # save the best model tqdm.write("\t saving best model...") bestsaver.save(sess, config.save_dir_best_model, global_step=global_step) tqdm.write("\t done.") finalperf = evalperf val_perf.append(evalperf) batchIdx, batch_data = batch batch_data = batch_data.data # each batch is ['pos'],['neg'] # hard negative mining # pair each pos image with max_similarity(pos_img,neg_sent) # and pos sent with max_similarity(pos_sent,neg_img) assert len(batch_data['pos']) == len(batch_data['neg']) # TODO: select hn_num of negative here, save computation? alldata = batch_data['pos'] + batch_data[ 'neg'] # (imgid,sent,sent_c) # 1. get all pos and neg's image and sentence embeddding all_imgs = list(set([one[0] for one in alldata])) imgid2idx = {} for imgid in all_imgs: imgid2idx[imgid] = len(imgid2idx.keys()) # load the actual image feature matrix image_feats = load_feats(imgid2idx, train_data.shared, config) mining_batch = {} mining_batch['imgs'] = [one[0] for one in alldata] mining_batch['imgid2idx'] = imgid2idx mining_batch['imgidx2feat'] = image_feats mining_batch['data'] = [(one[1], one[2]) for one in alldata ] # a list of (sent,sent_c) # mining_batch, N_pos+N_neg z_u, z_v = tester.step(sess, (batchIdx, Dataset(mining_batch, "test", shared=train_data.shared, is_train=False, imgfeat_dim=config.feat_dim))) assert len(z_u) == len(z_v), (len(z_u), len(z_v)) z_u_pos = z_u[:len(batch_data['pos'])] z_v_pos = z_v[:len(batch_data['pos'])] z_u_neg = z_u[len(batch_data['pos']):] z_v_neg = z_v[len(batch_data['pos']):] assert len(z_u_pos) == len(z_v_pos) == len(z_u_neg) == len( z_v_neg) == len(batch_data['pos']), (len(z_u_pos), len(z_v_pos), len(z_u_neg), len(z_v_neg), len(batch_data['pos'])) # 2. ensemble (pos_img,pos_sent,neg_img,neg_sent) batch # 2.1 for each pos_img ,find the best neg_sent, posimg2negsentIdxs = { } # pos idx :0 -> N-1, ====> neg idx :0 ->N-1 possent2negimgIdxs = {} check_num = config.hn_num neg_idxs = range(len(z_u_neg)) for i in xrange(len(batch_data['pos'])): """ pos_img_vec = z_v_pos[i] # [hop+1,d] check_neg_idxs = random.sample(neg_idxs,check_num) simis = np.zeros(check_num,dtype="float") for j,check_neg_idx in enumerate(check_neg_idxs): neg_sent_vec = z_u_neg[check_neg_idx] simis[j] = np.sum(pos_img_vec*neg_sent_vec) posimg2negsentIdxs[i] = check_neg_idxs[np.argmax(simis)] pos_sent_vec = z_v_pos[i] # [hop+1,d] check_neg_idxs = random.sample(neg_idxs,check_num) simis = np.zeros(check_num,dtype="float") for j,check_neg_idx in enumerate(check_neg_idxs): neg_img_vec = z_v_neg[check_neg_idx] simis[j] = np.sum(pos_sent_vec*neg_img_vec) possent2negimgIdxs[i] = check_neg_idxs[np.argmax(simis)] """ pos_img_vec = z_v_pos[i] # [hop+1,d] pos_sent_vec = z_v_pos[i] # [hop+1,d] check_neg_idxs = random.sample(neg_idxs, check_num) simis = np.zeros((2, check_num), dtype="float") for j, check_neg_idx in enumerate(check_neg_idxs): neg_sent_vec = z_u_neg[check_neg_idx] neg_img_vec = z_v_neg[check_neg_idx] simis[0, j] = np.sum(pos_img_vec * neg_sent_vec) simis[1, j] = np.sum(pos_sent_vec * neg_img_vec) posimg2negsentIdxs[i] = check_neg_idxs[np.argmax(simis[0])] possent2negimgIdxs[i] = check_neg_idxs[np.argmax(simis[1])] new_batch = {"data": []} imgids = {} for i in xrange(len(batch_data['pos'])): thisPos = batch_data['pos'][i] negImg = batch_data['neg'][possent2negimgIdxs[i]][0] negSent = batch_data['neg'][posimg2negsentIdxs[i]] thisNeg = (negImg, negSent[1], negSent[2]) new_batch['data'].append((thisPos, thisNeg)) # get all imageid imgids[thisPos[0]] = 1 imgids[thisNeg[0]] = 1 # no need to get feature again #imgid2idx = {} #for imgid in imgids: # imgid2idx[imgid] = len(imgid2idx.keys()) #image_feats = load_feats(imgid2idx,train_data.shared,config) new_batch['imgid2idx'] = imgid2idx new_batch['imgidx2feat'] = image_feats batch = batchIdx, Dataset(new_batch, "train", shared=train_data.shared, is_train=True, imgfeat_dim=config.feat_dim) loss, train_op = trainer.step(sess, batch) #print mcb1 #print "-"*40 if global_step % save_period != 0: # time to save model saver.save(sess, config.save_dir_model, global_step=global_step) str_ = "best eval on val %s: %s at %s step, final step %s %s is %s" % ( metric, best[metric], best['step'], global_step, metric, finalperf[metric]) print str_ self_summary_strs.append(str_) if config.write_self_sum: f = open(config.self_summary_path, "w") f.writelines("%s" % ("\n".join(self_summary_strs))) f.close() if config.record_val_perf: pickle.dump(val_perf, open(config.val_perf_path, "wb"))
parser = argparse.ArgumentParser() parser.add_argument('config_path') parser.add_argument('--fold', type=int) parser.add_argument('--training', action='store_true') args = parser.parse_args() with open(args.config_path, 'r') as f: cfg = json.load(f) config = Config(**cfg) skip_folds = [] if args.fold is not None: skip_folds = [i for i in range(4) if i != int(args.fold)] test = not args.training config = update_config(config, dataset_path=os.path.join(config.dataset_path, 'test' if test else 'train')) paths = { 'masks': os.path.join(config.dataset_path, 'masks2m'), 'images': os.path.join(config.dataset_path, 'images') } fn_mapping = { 'masks': lambda name: os.path.splitext(name)[0] + '.png' } image_suffix = 'img' def train_roads(): ds = ReadingImageProvider(RawImageType, paths, fn_mapping, image_suffix=image_suffix)
def main(): # Loading configuration config = params.params # Parsing command line parameters parser = argparse.ArgumentParser(description='Parametrized sample generation from GAN weights.') # -> Training parameters: parser.add_argument('--num_batches', metavar='nbatches', type=int, nargs=1, default=[1], help='Number of batches per class to train the classifier with ' '(default: %(default)s)') parser.add_argument('--batch_size', metavar='batch_size', type=int, nargs=1, default=[64], help='Size of each batch (same for generation/filtering/training, ' 'default: %(default)s)') parser.add_argument('--epochs', metavar='epochs', type=int, nargs=1, default=[10], help='Number of epochs to train the classifier for ' '(default: %(default)s)') # -> Input/Output: parser.add_argument('--model', metavar='model', type=str, nargs=1, help='Weights file to use for the GAN (of the form: ./weights/model_name.pth)') parser.add_argument('--classifier_model', metavar='class_model', type=str, nargs=1, default=['resnet20'], help='Weights file to use for the filtering classifier (of the form: ./classifiers/weights/class_model_name.pth)') parser.add_argument('--ofile', metavar='ofile', type=str, nargs=1, default=["trained_net"], help='Output file name ' '(default: %(default)s)') parser.add_argument('--num_workers', metavar='num_workers', type=float, nargs=1, default=[1], help='Number of workers to use for the dataloader.' '(default: %(default)s)') # -> Methods and parameters: parser.add_argument('--threshold', metavar='threshold', type=float, nargs=1, default=[0.9], help='Threshold probability for filtering ' '(default: %(default)s)') parser.add_argument('--truncate', metavar='truncate', type=float, nargs=1, default=[None], help='Sample latent z from a truncated normal ' '(default: no truncation).') parser.add_argument('--fixed_dset', action='store_true', help='Use a fixed generated dataset for training ' '(of size: batch_size x num_batches x num_classes, ' 'default: False)') parser.add_argument('--transform', action='store_true', help='Apply image transformations to generated images ' '(default: False)') parser.add_argument('--filter_samples', action='store_true', help='Enable classifier-filtering of generated images ' '(default: False)') # -> Multi-GANs stuff: parser.add_argument('--multi_gans', metavar='multi_gans', type=int, nargs=1, default=[None], help='Sample using multiple GANs ' '(default: %(default)s)') parser.add_argument('--gan_weights', metavar='gan_weights', type=float, nargs='+', default=[None], help='Specify weights for each GAN ' '(default: sample from each GAN with equiprobability)') args = vars(parser.parse_args()) # Values: num_batches = args['num_batches'][0] batch_size = args['batch_size'][0] model_name = args['model'][0] class_model_name = args['classifier_model'][0] ofile = args['ofile'][0] threshold = args['threshold'][0] num_workers = args['num_workers'][0] epochs = args['epochs'][0] trunc_norm = args['truncate'][0] multi_gans = args['multi_gans'][0] gan_weights = args['gan_weights'] if gan_weights[0] is not None and multi_gans != len(gan_weights): print('The list of GAN weights should specify weights for each GAN.') else: gan_weights = None # Toggles: fixed_dset = args['fixed_dset'] transform = args['transform'] filter_samples = args['filter_samples'] # Updating config object utils.update_config(config) run(config, num_batches, batch_size, model_name, class_model_name, ofile, threshold, num_workers, epochs, multi_gans, gan_weights, trunc_norm, fixed_dset, transform, filter_samples)
def main(): # Loading configuration config = params.params # Parsing command line parameters parser = argparse.ArgumentParser( description='Parametrized sample generation from GAN weights.') parser.add_argument('--num_samples', metavar='nsamples', type=int, nargs=1, default=[10], help='Number of samples to generate ' '(default: %(default)s)') parser.add_argument('--ofile', metavar='ofile', type=str, nargs=1, default=["samples"], help='Output file name ' '(default: %(default)s)') parser.add_argument( '--model', metavar='model', type=str, nargs=1, help='Model name to use (with weights in ./weights/model_name') parser.add_argument('--class', metavar='class', type=int, nargs=1, default=[None], help='Class to sample from (in [O,k-1] for k classes, ' 'default: sample [num_samples/k] for all classes.)') parser.add_argument('--multi_gans', metavar='multi_gans', type=int, nargs=1, default=[None], help='Sample using multiple GANs ' '(default: %(default)s)') parser.add_argument('--truncate', metavar='truncate', type=float, nargs=1, default=[None], help='Sample latent z from a truncated normal ' '(default: no truncation).') parser.add_argument( '--torch_format', action='store_true', help='Save sample archive using the torch format for samples' '(default: False)') parser.add_argument('--transform', action='store_true', help='Apply image transformations to generated images ' '(default: False)') parser.add_argument('--random_k', action='store_true', help='Sample classes randomly ' '(default: False)') args = vars(parser.parse_args()) # Values: num_samples = args['num_samples'][0] model_name = args['model'][0] ofile = args['ofile'][0] y_class = args['class'][0] multi_gans = args['multi_gans'][0] trunc_norm = args['truncate'][0] # Toggles: torch_format = args['torch_format'] transform = args['transform'] random_k = args['random_k'] # Updating config object utils.update_config(config) run(config, num_samples, model_name, ofile, y_class, torch_format, transform, multi_gans, trunc_norm, random_k)
def train(config): self_summary_strs = [] # summary string to print out for later val_perf = [] # summary of validation performance ############################################################################################ # flickr30k data # image : 31783 인데 29783개의 학습셋 # trainset : mRNN/flickr30K_train_list_mRNN.txt : 29783개 # valset : mRNN/flickr30K_val_list_mRNN.txt : 1000 # testset : mRNN/flickr30K_test_list_mRNN.txt : 1000 ############################################################################################ # first, read both data and filter stuff, to get the word2vec idx, train_data = read_data(config, 'train', config.load) # config.load = False config.imgfeat_dim = train_data.imgfeat_dim # print train_data.imgfeat_dim = train val_data = read_data(config, 'val', True, subset=False) # dev should always load model shared data(word2idx etc.) from train # now that the dataset is loaded , we get the max_word_size from the dataset # then adjust the max based on the threshold as well # also get the vocab size config_vars = vars(config) str_ = "threshold setting--\n" + "\t"+ " ,".join(["%s:%s"%(key,config_vars[key]) for key in config.thresmeta]) # threshold setting-- # sent_size_thres:200 ,word_size_thres:20 print str_ self_summary_strs.append(str_) # cap the numbers # max sentence word count etc. update_config(config,[train_data,val_data],showMeta=True) # all word num is <= max_thres str_ = "renewed ----\n"+"\t" + " ,".join(["%s:%s"%(key,config_vars[key]) for key in config.maxmeta]) # renewed ---- # max_sent_size:82 ,max_word_size:16 print str_ self_summary_strs.append(str_) # now we initialize the matrix for word embedding for word not in glove word2vec_dict = train_data.shared['word2vec'] # empty if not use pre-train vector # empty # 실제값은 {} word2idx_dict = train_data.shared['word2idx'] # this is the word not in word2vec # for finetuning or not using word2vec then it is all the word # word to index # 실제값은 {... 'dominoes': 5919, 'sleeves': 6909, 'laundromat': 5920, 'sash': 5921} # we are not fine tuning , so this should be empty; empty if not use pretrain vector # if finetuning , this will have glove idx2vec_dict = {word2idx_dict[word]:vec for word,vec in word2vec_dict.items() if word in word2idx_dict} # idx2vec_dict is empty # 실제값은 {} # len(idx2vec_dict):0,word_vocab_size:11798 print "len(idx2vec_dict):%s,word_vocab_size:%s"%(len(idx2vec_dict),config.word_vocab_size) # config.word_vocab_size = len(train_data.shared['word2idx']) # the word not in word2vec # so the emb_mat should all be a random vector # np.random.multivariate_normal gets mean of zero and co of 1 for each dim, like #>>> np.random.multivariate_normal(np.zeros(5),np.eye(5)) #array([-0.73663652, -1.16417783, -0.74083293, -0.80016731, 0.060182 ]) # random initial embedding matrix for new words # this will take a long time when vocab_size is 6k if not config.no_wordvec: # config.no_wordvec is True : 따라서, 여긴 진행하진 않는다. config.emb_mat = np.array([idx2vec_dict[idx] if idx2vec_dict.has_key(idx) else np.random.multivariate_normal(np.zeros(config.word_emb_size), np.eye(config.word_emb_size)) for idx in xrange(config.word_vocab_size)],dtype="float32") # for idx in xrange(config.word_vocab_size): # if idx2vec_dict.has_key(idx) # else: np.random.multivariate_normal(np.zeros(config.word_emb_size), np.eye(config.word_emb_size)) # print idx, idx2vec_dict.has_key(idx) # idx:11567, idx2vec_dict.has_key(idx) # , False 이면 밑의 random 초기화 변수를 선택한다. # print np.random.multivariate_normal(np.zeros(config.word_emb_size), np.eye(config.word_emb_size)) # print idx2vec_dict[idx] # from model import get_model model = get_model(config) # construct model under gpu0 #for var in tf.trainable_variables(): # print var.name,var.get_shape() #sys.exit() trainer = Trainer(model,config) tester = Tester(model,config) saver = tf.train.Saver(max_to_keep=5) # how many model to keep bestsaver = tf.train.Saver(max_to_keep=5) # just for saving the best model save_period = config.save_period # also the eval period # start training! # allow_soft_placement : tf will auto select other device if the tf.device(*) not available tfconfig = tf.ConfigProto(allow_soft_placement=True) tfconfig.gpu_options.allow_growth = True # this way it will only allocate nessasary gpu, not take all # or you can set hard limit #tfconfig.gpu_options.per_process_gpu_memory_fraction = 0.4 with tf.Session(config=tfconfig) as sess: # calculate total parameters totalParam = cal_total_param() str_ = "total parameters: %s"%(totalParam) print str_ self_summary_strs.append(str_) # config.load = False # config.load_best = False initialize(load=config.load,load_best=config.load_best,model=model,config=config,sess=sess) # the total step (iteration) the model will run last_time = time.time() # total / batchSize * epoch num_steps = int(math.ceil(train_data.num_examples/float(config.batch_size)))*config.num_epochs # num_steps = 34920 # train_data.num_examples = 148915 # config.batch_size = 256 # config.num_epochs = 60 # get_batches is a generator, run on the fly # there will be num_steps batch str_ = " batch_size:%s, epoch:%s, %s step every epoch, total step:%s,eval/save every %s steps"%(config.batch_size,config.num_epochs,math.ceil(train_data.num_examples/float(config.batch_size)),num_steps,config.save_period) print str_ # str_ = batch_size:256, epoch:60, 582.0 step every epoch, total step:34920,eval/save every 1000 steps self_summary_strs.append(str_) metric = "t2i_r@5" # TODO: change this? best = {metric:0.0,"step":-1} # remember the best eval acc during training finalperf = None isStart = True for batch in tqdm(train_data.get_batches(config.batch_size,no_img_feat=True,num_steps=num_steps),total=num_steps,ascii=True): # each batch has (batch_idxs,Dataset(batch_data, full_shared)) # batch_data has {"q":,"y":..."pidx2feat",.."photo_idxs"..} global_step = sess.run(model.global_step) + 1 # start from 0 # if load from existing model, save if first if (global_step % save_period == 0) or (config.load_best and isStart) or (config.load and isStart and (config.ignore_vars is None)): isStart=False tqdm.write("\tsaving model %s..."%global_step) saver.save(sess,config.save_dir_model,global_step=global_step) tqdm.write("\tdone") evalperf = evaluate(val_data,config,sess,tester) tqdm.write("\teval on validation:%s, (best %s:%s at step %s) "%(evalperf,metric,best[metric],best['step'])) # remember the best acc if(evalperf[metric] > best[metric]): best[metric] = evalperf[metric] best['step'] = global_step # save the best model tqdm.write("\t saving best model...") bestsaver.save(sess,config.save_dir_best_model,global_step=global_step) tqdm.write("\t done.") finalperf = evalperf val_perf.append(evalperf) # 여기는 모든 데이터 정보로부터 batch단위로 재구성한다. batchIdx, batch_data = batch batch_data = batch_data.data # each batch is ['pos'],['neg'] # hard negative mining # pair each pos image with max_similarity(pos_img,neg_sent) # and pos sent with max_similarity(pos_sent,neg_img) assert len(batch_data['pos']) == len(batch_data['neg']) # TODO: select hn_num of negative here, save computation? alldata = batch_data['pos'] + batch_data['neg'] # (imgid,sent,sent_c) # 1. get all pos and neg's image and sentence embeddding #for one in alldata: # print one[0] # image id # print one[1] # sentence_word # print one[2] # sentence_word_char # # ??? all_imgs의 길이는 512(156+256)이어야하는데, 이보다 조금 못미칠경우가 있다. # print len(alldata) = 512, 인데, # print len(all_imgs) = 508 ?? # 512, alldata 안에 확인결과 중복이 존재할수도 있다. # 이 중복은 positive data에서 발생, 이를 제외한 negative data를 만들었으니 당연~ # 그래서, 좀 바꿔야하지 않을까함 all_imgs = list(set([one[0] for one in alldata])) # so, batch image ids imgid2idx = {} for imgid in all_imgs: # batch image ids = 508이라면 imgid2idx[imgid] = len(imgid2idx.keys()) # print imgid, imgid2idx[imgid] # 2881441125 0 # ... # 51146082 472 # 2198484810 473 # 769934076 474 # 2874876837 475 # ... # 3681637675 507 # load the actual image feature matrix # conv image feature file load image_feats = load_feats(imgid2idx,train_data.shared,config) # only 학습셋이 아니라 테스트셋을 위한 재구축을 위한 batch # : alldata = batch_data['pos'] + batch_data['neg'] mining_batch = {} mining_batch['imgs'] = [one[0] for one in alldata] # image ids mining_batch['imgid2idx'] = imgid2idx mining_batch['imgidx2feat'] = image_feats # image id를 제외시킨 데이터 = a list of (sentence_word, sentence_word_char) mining_batch['data'] = [(one[1],one[2]) for one in alldata] # harg negative sample를 구하기 위한 작업 # mining_batch, N_pos+N_neg z_u, z_v = tester.step(sess,(batchIdx,Dataset(mining_batch, "test", shared=train_data.shared, is_train=False, imgfeat_dim=config.feat_dim))) # z_u : the output of the embedding for text, (512, 3, 512) # 2번째 dimension(=3)의 의미가 모지?? # z_v : the output of the embedding for images, (512, 3, 512) assert len(z_u) == len(z_v),(len(z_u),len(z_v)) z_u_pos = z_u[:len(batch_data['pos'])] z_v_pos = z_v[:len(batch_data['pos'])] z_u_neg = z_u[len(batch_data['pos']):] z_v_neg = z_v[len(batch_data['pos']):] assert len(z_u_pos) == len(z_v_pos) == len(z_u_neg) == len(z_v_neg) == len(batch_data['pos']),(len(z_u_pos),len(z_v_pos),len(z_u_neg),len(z_v_neg),len(batch_data['pos'])) # 2. ensemble (pos_img,pos_sent,neg_img,neg_sent) batch # 2.1 for each pos_img ,find the best neg_sent, posimg2negsentIdxs = {} # pos idx :0 -> N-1, ====> neg idx :0 ->N-1 possent2negimgIdxs = {} check_num = config.hn_num # 32 neg_idxs = range(len(z_u_neg)) # 256 # neg_idxs > [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255] # harg negative sample를 구하기 위한 작업 for i in xrange(len(batch_data['pos'])): pos_img_vec = z_v_pos[i] # [hop+1,d] pos_sent_vec = z_v_pos[i] # [hop+1,d] check_neg_idxs = random.sample(neg_idxs,check_num) simis = np.zeros((2,check_num),dtype="float") for j,check_neg_idx in enumerate(check_neg_idxs): neg_sent_vec = z_u_neg[check_neg_idx] neg_img_vec = z_v_neg[check_neg_idx] simis[0,j] = np.sum(pos_img_vec*neg_sent_vec) simis[1,j] = np.sum(pos_sent_vec*neg_img_vec) posimg2negsentIdxs[i] = check_neg_idxs[np.argmax(simis[0])] possent2negimgIdxs[i] = check_neg_idxs[np.argmax(simis[1])] new_batch = {"data":[]} imgids = {} for i in xrange(len(batch_data['pos'])): thisPos = batch_data['pos'][i] negImg = batch_data['neg'][possent2negimgIdxs[i]][0] negSent = batch_data['neg'][posimg2negsentIdxs[i]] thisNeg = (negImg,negSent[1],negSent[2]) new_batch['data'].append((thisPos,thisNeg)) # get all imageid imgids[thisPos[0]] = 1 imgids[thisNeg[0]] = 1 # no need to get feature again #imgid2idx = {} #for imgid in imgids: # imgid2idx[imgid] = len(imgid2idx.keys()) #image_feats = load_feats(imgid2idx,train_data.shared,config) new_batch['imgid2idx'] = imgid2idx new_batch['imgidx2feat'] = image_feats batch = batchIdx,Dataset(new_batch,"train",shared=train_data.shared,is_train=True,imgfeat_dim=config.feat_dim) loss,train_op = trainer.step(sess,batch) #print mcb1 #print "-"*40 if global_step % save_period != 0: # time to save model saver.save(sess,config.save_dir_model,global_step=global_step) str_ = "best eval on val %s: %s at %s step, final step %s %s is %s"%(metric,best[metric],best['step'], global_step,metric,finalperf[metric]) print str_ self_summary_strs.append(str_) if config.write_self_sum: f = open(config.self_summary_path,"w") f.writelines("%s"%("\n".join(self_summary_strs))) f.close() if config.record_val_perf: pickle.dump(val_perf,open(config.val_perf_path,"wb"))
def initial_setup(): if current_user.is_authenticated: return redirect(url_for("warden.warden_page")) page = request.args.get("page") # initial setup will cycle through different pages if page is None or page == 'welcome' or page == '1': # Generate a random API key for Alphavantage import secrets key = secrets.token_hex(15) current_app.settings['API']['alphavantage'] = key update_config() return render_template("warden/welcome.html", title="Welcome to the WARden") if page == '2' or page == 'register': form = RegistrationForm() if form.validate_on_submit(): hash = generate_password_hash(form.password.data) user = User(username=form.username.data, password=hash) current_app.db.session.add(user) current_app.db.session.commit() login_user(user, remember=True) flash(f"Account created for {form.username.data}. User Logged in.", "success") return redirect("/initial_setup?page=3&setup=True") return render_template("warden/register.html", title="Welcome to the WARden | Register", form=form) if page == '3' or page == 'specter_connect': # First let's check where we can connect with Tor tor_ports = ['9050', '9150'] session = requests.session() # Use DuckDuckGo Onion address to test tor url = 'https://3g2upl4pq6kufc4m.onion' failed = True for PORT in tor_ports: session.proxies = { "http": "socks5h://0.0.0.0:" + PORT, "https": "socks5h://0.0.0.0:" + PORT, } try: session.get(url) session.close() failed = False except Exception: failed = True if not failed: current_app.settings['TOR']['port'] = PORT update_config() break if failed: flash("Tor does not seem to be running in any ports...", "warning") # Maybe Specter is already running? try: if current_app.specter.home_parser()['alias_list'] != []: flash(f"Succesfuly connected to Specter Server at {current_app.specter.base_url}") except Exception: pass return redirect(url_for("warden.warden_page"))
print "Tempdir is:", tempdir mainLogger.info("Checking all Dependencies...") mainLogger.debug("truerypt binary is {}".format( parser.get('truecrypting', 'tc_binary'))) dependency_check( [parser.get('truecrypting', 'tc_binary'), "--text", "--version"]) mainLogger.info("Configuring...") # # Setting Parameters given from argparse # update_config("DEFAULT", "device", args.device) update_config("thunderbird_linux", "version", args.thunder) update_config("thunderbird_windows", "version", args.thunder) update_config("thunderbird_mac", "version", args.thunder) update_config("vidalia_linux", "version", args.vidalia) update_config("vidalia_windows", "version", args.vidalia) update_config("vidalia_mac", "version", args.vidalia) update_config("DEFAULT", "container_name", args.container_name) update_config("truecrypting", "size", args.container_size) print "=" * 60, "\nMounting and Truecrypting\n", "=" * 60 # # Use an USB-Stick given by a Parameter or a detected one: #