def run_kmeans(X, plot=False): """ My solution: for i in range(len(K)): for j in range(len(seed)): mixture, post = common.init(X, K[i], seed[j]) mixture, post, cost = kmeans.run(X, mixture, post) print("K = {}, seed = {}, cost = {}".format(K[i], seed[j], cost)) if plot: common.plot(X, mixture, post, "K={}, seed={}".format(K[i], seed[j])) """ # Instructor's solution: for K in range(1, 5): min_cost = None best_seed = None for seed in range(0, 5): mixture, post = common.init(X, K, seed) mixture, post, cost = kmeans.run(X, mixture, post) if min_cost is None or cost < min_cost: min_cost = cost best_seed = seed mixture, post = common.init(X, K, best_seed) mixture, post, cost = kmeans.run(X, mixture, post) title = "K-means for K=, seed=, cost=".format(K, best_seed, min_cost) print(title) common.plot(X, mixture, post, title)
def test_seeds(X, K): print("\n############## KMEAN K=" + str(K) + " ###############") mixture0, post0 = common.init(X, K, 0) mixture1, post1 = common.init(X, K, 1) mixture2, post2 = common.init(X, K, 2) mixture3, post3 = common.init(X, K, 3) mixture4, post4 = common.init(X, K, 4) cost0 = kmeans.run(X, mixture0, post0)[2] cost1 = kmeans.run(X, mixture1, post1)[2] cost2 = kmeans.run(X, mixture2, post2)[2] cost3 = kmeans.run(X, mixture3, post3)[2] cost4 = kmeans.run(X, mixture4, post4)[2] print("K=" + str(K) + " seed=0 : cost=" + str(cost0)) print("K=" + str(K) + " seed=1 : cost=" + str(cost1)) print("K=" + str(K) + " seed=2 : cost=" + str(cost2)) print("K=" + str(K) + " seed=3 : cost=" + str(cost3)) print("K=" + str(K) + " seed=4 : cost=" + str(cost4)) naive_em_estimate0 = naive_em.run(X, mixture0, post0) naive_em_estimate1 = naive_em.run(X, mixture1, post1) naive_em_estimate2 = naive_em.run(X, mixture2, post2) naive_em_estimate3 = naive_em.run(X, mixture3, post3) naive_em_estimate4 = naive_em.run(X, mixture4, post4) print("K=" + str(K) + " seed=0 : likelihood=" + str(naive_em_estimate0[2])) print("K=" + str(K) + " seed=1 : likelihood=" + str(naive_em_estimate1[2])) print("K=" + str(K) + " seed=2 : likelihood=" + str(naive_em_estimate2[2])) print("K=" + str(K) + " seed=3 : likelihood=" + str(naive_em_estimate3[2])) print("K=" + str(K) + " seed=4 : likelihood=" + str(naive_em_estimate4[2]))
def test(self): try: this_dir = os.path.dirname(os.path.abspath(__file__)) init(visualiser_args=['--maps', '{}/RRT_test_map_2D.py'.format(this_dir)]) graphics_test() finally: destroy()
def run(): """ :param app_name: HPC application :param perf_coln: performance name to be optimized :param num_core: number of CPU cores :param num_node: number of computing nodes :param rand_seed: random seed :param num_smpl: number of samples :param pool_size: pool size :param num_iter: number of iterations :param prec_rand: precentage of random samples """ try: cm.init() app_name = cm.app_name perf_coln = cm.perf_coln num_smpl = cm.num_smpl pool_size = cm.pool_size num_iter = cm.num_iter prec_rand = cm.prec_rand if (app_name == "lv"): conf_colns = data.lv_conf_colns elif (app_name == "hs"): conf_colns = data.hs_conf_colns num_rand = int(num_smpl * prec_rand) nspi = int((num_smpl - num_rand) / num_iter) # pool_df = data.gen_smpl(app_name, pool_size) # conf_df = pool_df.head(num_rand) conf_df = data.gen_smpl(app_name, num_rand) train_df = cm.measure_perf(conf_df) for iter_idx in range(num_iter): num_curr = num_smpl - nspi * (num_iter - 1 - iter_idx) pool_df = data.gen_smpl(app_name, pool_size) pred_top_smpl = learn.whl_pred_top_eval(train_df, pool_df, conf_colns, perf_coln, num_smpl, 0) pred_top_smpl = pred_top_smpl.sort_values([perf_coln]).reset_index(drop=True) new_conf_df = pred_top_smpl[conf_colns].head(nspi) conf_df = tool.df_union(conf_df, new_conf_df) last = nspi while (conf_df.shape[0] < num_curr): last = last + 1 new_conf_df = pred_top_smpl[conf_colns].head(last) conf_df = tool.df_union(conf_df, new_conf_df) new_train_df = cm.measure_perf(new_conf_df) train_df = tool.df_union(train_df, new_train_df) data.df2csv(train_df, app_name + "_train.csv") mdl_chk, mdl = learn.train_mdl_chk(train_df, conf_colns, perf_coln) top_df = cm.find_top('ALe', (mdl_chk, mdl, ), conf_colns, perf_coln, train_df) cm.test(train_df, conf_colns, perf_coln) cm.finish(train_df, top_df) except: traceback.print_exc()
def test_common_init(self): layer_widths=[32, 16, 8, 7, 6, 5, 4, 3, 2, 1] centroids= [2, 8, 32,32,64,16,16,16,16,4] cm.init(centroids=centroids, video_file="moving_square.avi", learn_rate=0.05, layer_widths=layer_widths, img_width=512 )
def __init__(self, parent): """Initializes the Background tab. Args: parent (App(QDialog)): Object corresponding to the parent UI element. """ self.parent = parent self.tag = "bg" self.dlg = parent.dlg common.init(self)
def __init__(self, parent): """Initializes the DepthEstimation tab. Args: parent (App(QDialog)): Object corresponding to the parent UI element. """ self.parent = parent self.tag = "depth" self.dlg = parent.dlg common.init(self)
def __init__(self, parent): """Initializes the Export tab. Args: parent (App(QDialog)): Object corresponding to the parent UI element. """ self.parent = parent self.tag = "export" self.dlg = parent.dlg common.init(self) self.initialize_viewer_buttons()
def run_kmeans(): for K in range(1, 5): min_cost = None best_seed = None for seed in range(0, 5): mixture, post = common.init(X, K, seed) mixture, post, cost = kmeans.run(X, mixture, post) if min_cost is None or cost < min_cost: min_cost = cost best_seed = seed mixture, post = common.init(X, K, best_seed) mixture, post, cost = kmeans.run(X, mixture, post) title = "K-means for K={}, seed={} , cost= {}".format(K, best_seed, min_cost) common.plot(X, mixture, post, title)
def run_kmean(X): for K in [1,2,3,4]: cost_list = [] for seed in range(5): mixture, post = common.init(X, K, seed) mixture, post, cost = kmeans.run(X, mixture, post) cost_list.append(cost) #common.plot(X, mixture, post, "{} means with seed{}".format(K, seed)) print("The cost of {} cluster is".format(K), min(cost_list)) best_seed = np.argmin(cost_list) for seed_ in [best_seed]: mixture, post = common.init(X, K, int(seed_)) mixture, post, cost = kmeans.run(X, mixture, post) common.plot(X, mixture, post, "{} means with seed{}".format(K, seed_)) return "Done"
def main(): print("Booting") common.init() telegram_bot_token="" twitter_consumer_key="" twitter_consumer_secret="" twitter_access_token="" tiwtter_access_secret="" # READ API TOKENS FROM FILE with open('tokens.json') as token_file: token_data = json.load(token_file) telegram_bot_token = token_data['telegram_bot_token'] twitter_consumer_key = token_data['consumer_key'] twitter_consumer_secret = token_data['consumer_secret'] twitter_access_token = token_data['access_token'] tiwtter_access_secret = token_data['access_token_secret'] follow_list = [] with open('follow_list.json') as follow_list_file: follow_data = json.load(follow_list_file) follow_list = follow_data['follow_list'] #AUTH auth = tweepy.OAuthHandler(twitter_consumer_key, twitter_consumer_secret) auth.set_access_token(twitter_access_token, tiwtter_access_secret) api = tweepy.API(auth) follow_list_ids = [] for item in follow_list: user = api.get_user(item) follow_list_ids.append(str(user.id)) stream = "" try: #SET UP STREAM streamListener = TweetsStreamListener() stream = tweepy.Stream(auth=api.auth, listener=streamListener) stream.filter(follow=follow_list_ids, async=True) bot.bot_main(telegram_bot_token) except KeyboardInterrupt: print("KeyboardInterrupt") stream.disconnect() print("Disconnected from stream")
def main(): global path logger = common.configLogger() common.init(logger) try: if "/build" not in path: msg = ("ERROR: exec_command() -" " build all faild.\n\nOutput msg:\n{}") msgf = msg.format(path) logger.info(msgf) raise AssertionError(msgf) bakeSrcCode(logger) except: logger.info(traceback.print_exc())
def main(): common.parser().add_option("--indb", dest="indb", help="Input database") if not common.init(__file__): exit(2) progname = common.progname() indb = common.options().indb if not indb: indb = common.testdir() + "/pgn/Kramnik.pgn" temppgn1 = common.tmpdir() + "/copydb1.pgn" temppgn2 = common.tmpdir() + "/copydb2.pgn" tempcfdb1 = common.tmpdir() + "/copydb1.cfdb" if run(1, indb, tempcfdb1) and run(2, tempcfdb1, temppgn1) and run(3, indb, temppgn2): # temppgn1 and temppgn2 should be identical if os.path.exists(temppgn1) and os.path.exists(temppgn2): lines1 = open(temppgn1, "U").readlines() lines2 = open(temppgn2, "U").readlines() diff = difflib.unified_diff(lines1, lines2) different = False first = True for line in diff: if first: print "{0}: PGN files {1} and {2} are different:".format(progname, temppgn1, temppgn2) different = True first = False print line if not different: print "{0}: Test successful".format(progname) else: print "{0}: Failed; no output files to compare".format(progname)
def r_tag(guid): client = init() tag = client.get_tag(guid) if not tag: abort(404) data = globaldata() data.tag = tag if data.user: modify_tag(client, data) data.cloud = tagcloud([guid]) data.q = tag.name data.tagtypes = client.metalist(u"tagtypes") def get_impl(rev): res = [] for i in client.tag_implies(guid, reverse=rev) or []: name = tagname(i.guid) res.append(ImplicationTupleWithName(*(i + (name,)))) return res data.implies_tags = get_impl(False) data.implied_by_tags = get_impl(True) order = "group" if tag.ordered else "aaaaaa-aaaac8-faketg-bddate" props = DotDict() posts = client.search_post(guids=[guid], order=order, range=[0, per_page - 1], wanted=wanted, props=props) data.posts = posts data.result_count = props.result_count data.page = 0 if posts: data.pagelink = makelink(u'search', (u'q', tag.name)) data.pages, data.rels = pagelinks(data.pagelink, 0, data.result_count) return data
def setupCommon(): from datetime import datetime bottle.request.session = bottle.request.environ['beaker.session'] try: user = aaa.current_user.username except: user = '******' workDB = bottle.request.query.workDB if (not workDB) and ('workDB' in bottle.request.session): workDB = bottle.request.session['workDB'] matchDB = bottle.request.query.matchDB if (not matchDB) and ('matchDB' in bottle.request.session): matchDB = bottle.request.session['matchDB'] if workDB: #KOLLA possibly store mongoClient in session? Param to init? common.config = common.init(workDB, matchDBName = matchDB) bottle.request.session['workDB'] = workDB bottle.request.session['matchDB'] = matchDB bottle.request.session.save() if conf.config.logging and ('action' in bottle.request.url or 'runProg' in bottle.request.url): rec = {'type': 'admin', 'time': time.time(), 'workDB': workDB, 'matchDB': matchDB, 'url': bottle.request.url, 'from': bottle.request.remote_addr, 'user': user} common.config['originalData'].insert_one(rec) bottle.response.set_header("Cache-Control", "no-cache") #print user, bottle.request.remote_addr, str(datetime.now()), bottle.request.url logging.info('%s %s %s', user, bottle.request.remote_addr, bottle.request.url)
def test_em(): init_mixture, post = common.init(X, K, seed) mixture, post, c = em.run(X, init_mixture, post) prediction = em.fill_matrix(X, mixture) print(c) print(common.rmse(prediction, X_gold))
def run_naive_em(X): for K in [1,2,3,4]: likelihood_ls = [] for seed in range(5): mixture, post = common.init(X, K, seed) mixture, post, LL = naive_em.run(X,mixture,post) likelihood_ls.append(LL) print("The likelihood of {} cluster is".format(K), max(likelihood_ls)) best_seed = np.argmax(likelihood_ls) for seed_ in [best_seed]: mixture, post = common.init(X, K, int(seed_)) mixture, post, LL= naive_em.run(X, mixture, post) common.plot(X, mixture, post, "{} mixtures with seed{}".format(K, seed_)) return "Done"
def run_naive_em(): for K in range(1, 5): max_ll = None best_seed = None for seed in range(0, 5): mixture, post = common.init(X, K, seed) mixture, post, ll = naive_em.run(X, mixture, post) if max_ll is None or ll > max_ll: max_ll = ll best_seed = seed mixture, post = common.init(X, K, best_seed) mixture, post, ll = naive_em.run(X, mixture, post) title = "EM for K={}, seed={}, ll={}".format(K, best_seed, ll) print(title) common.plot(X, mixture, post, title)
def main(): common.parser().add_option("--epdfile", dest = "epdfile", help = "Input EPD file") common.parser().add_option("--timecontrol", dest="timecontrol", default = "10s", help = "Time control") common.parser().add_option("--engine", dest="engine", help = "The engine to use to process the EPD file") if not common.init(__file__): exit(2) progname = common.progname() debuglog = True logcomms = False logfile = common.tmpdir() + "/ccore.log" configfile = common.configfile() epdfile = common.options().epdfile if not epdfile: usage() timecontrol = common.options().timecontrol engine = common.options().engine if not engine: engine = common.engine1() if os.path.exists(logfile): os.remove(logfile) # Take a copy of the database cmdline = common.ccore() if debuglog: cmdline += " --debuglog true" if logcomms: cmdline += " --logcomms true" cmdline += " -c {0} -l {1} -e {2} -t {3} processepd {4}".format(configfile, logfile, epdfile, timecontrol, engine); if common.runccore(cmdline): common.checkLogfile(logfile)
def train(): args = init() global_epoch = args.init_epoch global_steps = 0 gen_path = args.outpath if not isdir(gen_path): mkdir(gen_path) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) train_writer = tf.summary.FileWriter(args.model_root, sess.graph) for e in range(args.init_epoch, args.epochs): global_epoch += 1 for i, (data, y) in enumerate( data_generator(args.data_root, args.label_path)): if y.size == 0: continue #data[np.where(data < 255)] = 0. #data = data.astype(np.float32) data = data.reshape(batch, size) _, _dis_loss, summary = sess.run([opt_dis, dis_loss, merged], feed_dict={ real_x: data, gen_data: rand_sample([batch, size]) }) train_writer.add_summary(summary, global_steps) _, _gen_loss, summary = sess.run([opt_gen, gen_loss, merged], feed_dict={ real_x: data, gen_data: rand_sample([batch, size]) }) train_writer.add_summary(summary, global_steps) if _dis_loss is None or _gen_loss is None: print('[{}/{}] dis_loss:{} gen_loss:{}'.format( global_epoch, i + 1, _dis_loss, _gen_loss), flush=True) return print('[{}/{}] dis_loss:{:.4} gen_loss:{:.4}'.format( global_epoch, i + 1, _dis_loss, _gen_loss), flush=True) sample = sess.run( fake_x, feed_dict={gen_data: rand_sample([batch, size])}) ind = i % gen_sample_nr i // gen_sample_nr and np.save('{}/{}'.format(gen_path, ind), sample) or None global_steps += 1
def run_matrix_completion(): K = 12 seed = 1 mixture, post = common.init(X, K, seed) mixture, post, ll = em.run(X, mixture, post) X_pred = em.fill_matrix(X, mixture) X_gold = np.loadtxt('netflix_complete.txt') print("RMSE:", common.rmse(X_gold, X_pred))
def colormap(force=False, view=False): ''' return cached file or create a new one ''' logging.info('colormap() called, force: %s, view: %s', force, view) init() print('content-type: text/json\r\n\r\n', end='') if os.path.exists(MAPFILE) and not force: logging.info('returning cached color map') with open(MAPFILE) as infile: colormap = infile.read() else: logging.info('creating new color map') colormap = json.dumps(create_colormap(view)) with open(MAPFILE, 'w') as outfile: outfile.write(colormap) print(colormap, end='')
def main(): common.parser().add_option("--indb", dest = "indb", help = "Input database") common.parser().add_option("--outdb", dest = "outdb", help = "Output database") common.parser().add_option("--firstgame", dest="firstgame", help = "First game in input database to analyze") common.parser().add_option("--lastgame", dest="lastgame",help = "First game in input database to analyze") common.parser().add_option("--ecofile", dest = "ecofile", help = "The (.cfdb) database containing the ECO classification") if not common.init(__file__): exit(2) progname = common.progname() debuglog = True logcomms = False logfile1 = common.tmpdir() + "/ccore1.log" logfile2 = common.tmpdir() + "/ccore2.log" configfile = common.configfile() indb = common.options().indb if not indb: indb = common.testdir() + "/pgn/Boris_Spassky.pgn" outdb = common.options().outdb if not outdb: outdb = common.tmpdir() + "/classified.cfdb" if os.path.exists(outdb): os.remove(outdb) ecofile = common.options().ecofile if not ecofile: ecofile = common.testdir() + "/cfdb/eco.cfdb" firstgame = common.options().firstgame lastgame = common.options().lastgame if os.path.exists(logfile1): os.remove(logfile1) if os.path.exists(logfile2): os.remove(logfile2) # Take a copy of the database cmdline = common.ccore() if debuglog: cmdline += " --debuglog true" if logcomms: cmdline += " --logcomms true" if firstgame: cmdline += " -n {0}".format(firstgame) if lastgame: cmdline += " -N {0}".format(lastgame) cmdline += " -l {0} -i {1} -o {2} copydb".format(logfile1, indb, outdb); if common.runccore(cmdline): common.checkLogfile(logfile1) # And then classify the copy cmdline = common.ccore() if debuglog: cmdline += " --debuglog true" if logcomms: cmdline += " --logcomms true" cmdline += " -l {0} -i {1} -E {2} classify".format(logfile2, outdb, ecofile) if common.runccore(cmdline): common.checkLogfile(logfile2)
def best_run_em(X): K = 12 dict = {} for seed in range(5): np.random.seed(seed) mixture, post = common.init(X, K, seed) mixture, post, LL = em.run(X, mixture, post) dict[LL] = (mixture, seed) return dict[min(dict.keys())]
def best_run_em(X): K = 12 dict = {} likelihood_ls = [] for seed in range(5): mixture, post = common.init(X, K, seed) mixture, post, LL = em.run(X, mixture, post) dict[LL] = mixture return dict[min(dict.keys())]
def test_k12(): lls = [] for s in [0, 1, 2, 3, 4]: print(s) init_mixture, post = common.init(X, 12, s) model = em.run(X, init_mixture, post) lls.append(model) m, p, l = max(lls, key=lambda x: x[-1]) prediction = em.fill_matrix(X, m) return common.rmse(prediction, X_gold)
def select_best_bic(X): bic_ls = [] for K in [1,2,3,4]: likelihood_ls = [] bic_ls_seed = [] for seed in range(5): mixture, post = common.init(X, K, seed) mixture, post, LL = naive_em.run(X,mixture,post) likelihood_ls.append(LL) bic_ls_seed.append(common.bic(X, mixture, LL)) best_seed = np.argmax(bic_ls_seed) mixture, post = common.init(X, K, int(best_seed)) mixture, post, LL = naive_em.run(X, mixture, post) bic_ls.append(common.bic(X,mixture,LL)) print("The best K is {} with bic {}".format(np.argmax(bic_ls)+1, max(bic_ls))) return "Done"
def test_incomplete_em(): for k_s in [1, 12]: lps = [] for s in [0, 1, 2, 3, 4]: print(k_s, s) init_mixture, post = common.init(X, k_s, s) model = em.run(X, init_mixture, post) lps.append(model) best = max(lps, key=lambda x: x[-1]) print(best[-1])
def test_em_seeds(X, K): print("\n############## EM K=" + str(K) + " ###############") mixture0, post0 = common.init(X, K, 0) mixture1, post1 = common.init(X, K, 1) mixture2, post2 = common.init(X, K, 2) mixture3, post3 = common.init(X, K, 3) mixture4, post4 = common.init(X, K, 4) cost0 = em.run(X, mixture0, post0)[2] cost1 = em.run(X, mixture1, post1)[2] cost2 = em.run(X, mixture2, post2)[2] cost3 = em.run(X, mixture3, post3)[2] cost4 = em.run(X, mixture4, post4)[2] print("K=" + str(K) + " seed=0 : likelihood=" + str(cost0)) print("K=" + str(K) + " seed=1 : likelihood=" + str(cost1)) print("K=" + str(K) + " seed=2 : likelihood=" + str(cost2)) print("K=" + str(K) + " seed=3 : likelihood=" + str(cost3)) print("K=" + str(K) + " seed=4 : likelihood=" + str(cost4))
def run_em(X): for K in [1, 12]: likelihood_ls = [] for seed in range(5): mixture, post = common.init(X, K, seed) mixture, post, LL = em.run(X, mixture, post) likelihood_ls.append(LL) print("The likelihood of {} cluster is".format(K), max(likelihood_ls)) return "Done"
def train(): args = init() global_epoch = args.init_epoch global_steps = 0 gen_path = args.outpath if not isdir(gen_path): mkdir(gen_path) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) train_writer = tf.summary.FileWriter(args.model_root, sess.graph) for e in range(args.init_epoch, args.epochs): global_epoch += 1 for i, (data, y) in enumerate( data_generator(args.data_root, args.label_path)): if y.shape == 0: continue data = data.reshape(batch, x_dim) _, _vae_loss, summary = sess.run([opt, vae_loss, merged], feed_dict={ real_x: data, z: np.random.randn( batch, z_dim) }) train_writer.add_summary(summary, global_steps) if _vae_loss is None: print('[{}/{}] vae_loss:{}'.format(global_epoch, i + 1, _vae_loss), flush=True) return print('[{}/{}] vae_loss:{:.4}'.format(global_epoch, i + 1, _vae_loss), flush=True) global_steps += 1 if i // gen_sample_nr != 0: continue sample = sess.run(fake_x, feed_dict={z: np.random.randn(batch, z_dim)}) ind = i % gen_sample_nr i // gen_sample_nr and np.save('{}/{}'.format(gen_path, ind), sample) or None print('model saved @ {}'.format(saver.save(sess, args.model_root)), flush=True)
def run_naive_em_with_bic(): max_bic = None for K in range(1, 5): max_ll = None best_seed = None for seed in range(0, 5): mixture, post = common.init(X, K, seed) mixture, post, ll = naive_em.run(X, mixture, post) if max_ll is None or ll > max_ll: max_ll = ll best_seed = seed mixture, post = common.init(X, K, best_seed) mixture, post, ll = naive_em.run(X, mixture, post) bic = common.bic(X, mixture, ll) if max_bic is None or bic > max_bic: max_bic = bic title = "EM for K={}, seed={}, ll={}, bic={}".format(K, best_seed, ll, bic) print(title) common.plot(X, mixture, post, title)
def test_naive_em(): for k in [1, 2, 3, 4]: para_list = [] for seed in [0, 1, 2, 3, 4]: gm, post = common.init(X, k, seed) mixture, p, cost = naive_em.run(X, gm, post) para_list.append((mixture, p, cost)) max_para = max(para_list, key=lambda x: x[2]) common.plot(X, max_para[0], max_para[1], 'EM on toy data with {k}'.format(k=k)) return max_para[0], max_para[1]
def ajax_completetag(): tag = request.query.q client = init() full_tag, alts = complete(client, tag) res = {} if full_tag[0] or alts: res["complete"] = full_tag[0] res["type"] = full_tag[1] if len(alts) > 20: alts = [] res["alts"] = alts return res
def run_matrix_completion(): K = 12 seed = 1 mixture, post = common.init(X, K, seed) (mu, var, p), post, ll = em.run(X, mixture, post) # print('Mu:\n' + str(mu)) # print('Var: ' + str(var)) # print('P: ' + str(p)) # print('post:\n' + str(post)) # print('LL: ' + str(ll)) X_pred = em.fill_matrix(X, common.GaussianMixture(mu, var, p)) X_gold = np.loadtxt('netflix_complete.txt') print("MAE:", common.mae(X_gold, X_pred))
def listdubl(): if not bottle.request.query.workDB: if 'workDB' in bottle.request.session: bottle.request.query.workDB = bottle.request.session['workDB'] else: return 'Databas I ej vald - programmet avslutas<br><a href="/">Tillbaka till startsida</a>' bottle.request.query.matchDB = bottle.request.query.workDB bottle.request.session['matchDB'] = bottle.request.query.workDB #need to re-init database collections common.config = common.init(bottle.request.query.workDB, matchDBName = bottle.request.query.matchDB) #test if workDB matched against itself dbOK = getDBselect('listDubl', bottle.request.query.workDB, bottle.request.session['activeUser'], bottle.request.session['directory']) if dbOK == 'No valid choices': #if not run match return runprog('match') #else show list from uiUtils import persDisp tit = 'Lista från alternativ dubblettkontroll' if bottle.request.params.sortNS: sortVal = 'checked' sorting = [('nodesim', -1),('sortDubl', -1)] else: sortVal = '' sorting = [('sortDubl', -1),('nodesim', -1)] page = int(bottle.request.params.pageNo or '1') prevnext = bottle.request.params.page or '' if prevnext == 'prev': page += -1 elif prevnext == 'next': page += 1 else: page = 1 if page <= 0: page = 1 rows = [['#',u'Namn/refId', u'Född', u'Död','Score/NodeSim', u'Namn/refId', u'Född', u'Död','Visa']] i = (page-1)*10 args = {'where': 'visa', 'what': '/view/persons', 'buttons': 'No'} # for mt in common.config['matches'].find({'nodesim': {'$gt': 0.3}}).sort([('nodesim', -1),('sortDubl', -1)]).limit( 50 ): tot = common.config['matches'].find({'nodesim': {'$gt': 0.3}}).count() for mt in common.config['matches'].find({'nodesim': {'$gt': 0.3}}).sort(sorting).skip((page-1)*10).limit( 10 ): #print mt['pwork']['name'],mt['pmatch']['name'], mt['sortDubl'], mt['nodesim'] i += 1 row = [str(i)] row.extend(persDisp(mt['pwork'])) row.append(str(mt['sortDubl'])+'<br>'+str(mt['nodesim'])) row.extend(persDisp(mt['pmatch'])) args['wid'] = str(mt['workid']) args['mid'] = str(mt['matchid']) row.append('<button onclick="doAction('+str(args)+')">Visa</button>') rows.append(row) return bottle.template('dubl', title = tit, page = page, tot = tot, prow=rows, sort=sortVal)
def ajax_tag(): client = init() tags = request.forms.tags name = request.forms.name if name: try: type = request.forms.type client.add_tag(tag_clean(name), type) tags = name except Exception: msg = u'Failed to create' m = request.forms.m.split() full = set() weak = set() remove = set() failed = [] for t in tags.split(): tag = client.find_tag(tag_clean(t)) if tag: p = tag_prefix(t) if p == "~": weak.add(tag) elif p == "-": remove.add(tag) else: full.add(tag) else: failed.append(t) res = {} msg = u'' if full or weak or remove: client.begin_transaction() for p in map(client.get_post, m): if not p: msg = u'Posts missing?' elif tag_post(p, full, weak, remove): p = client.get_post(p.md5) res[p.md5] = tags_as_html(p) client.end_transaction() if not res and not msg and not failed and not (name and name[0] == '-'): msg = u'Nothing to do?' res = dict(failed=u' '.join(failed), m=res, msg=msg) if failed: res["types"] = tagtypes() return res
def r_search(): data = globaldata() client = init() def parse_tag(name): res = client.parse_tag(name, comparison=True) if res: guid, cmp, val = res tag = client.get_tag(guid, with_prefix=True) return (tag, cmp, val) try: page = max(0, int(request.query.page)) except Exception: page = 0 q = request.query.q.strip() data.tagnames = qa = q.split() data.tags = map(parse_tag, qa) data.q = q = u' '.join(qa) data.cloud = [] data.result_count = 0 ta = [] for i, (tag, cmp, val) in enumerate(filter(None, data.tags)): if cmp: qa[i] = tag_prefix(qa[i]) + tag.name ta.append((tag, cmp, val)) if ta or not q: if data.user and request.query.ALL: range = [0, 1 << 31 - 1] page = -1 else: range = [per_page * page, per_page * page + per_page - 1] order = "aaaaaa-aaaac8-faketg-bddate" if ta and ta[0][0].ordered: order = "group" props = DotDict() ga = [(t.pguid, cmp, val) for t, cmp, val in ta] posts = client.search_post(guids=ga, order=order, range=range, wanted=wanted, props=props) print "mm" if posts: data.posts = posts data.result_count = props.result_count data.page = page data.pagelink = makelink(u'search', (u'q', q)) data.pages, data.rels = pagelinks(data.pagelink, page, data.result_count) data.cloud = tagcloud(ga) return data
def r_post_rotate(): m = request.forms.post rot = int(request.forms.rot or 0) assert rot in (0, 90, 180, 270) if rot: client = init() post = client.get_post(m, wanted=["rotate", "ext", "width", "height"]) props = DotDict() if rot in (90, 270): props.width, props.height = post.height, post.width prot = int(post.rotate) if prot == -1: prot = 0 assert prot in (0, 90, 180, 270) rot = (prot + rot) % 360 client.save_thumbs(m, None, post.ext, rot, True) props.rotate = rot client.modify_post(m, **props) redirect("post/" + m)
def main(): common.parser().add_option("--indb", dest = "indb", help = "Input database") common.parser().add_option("--outdb", dest = "outdb", help = "Output database") if not common.init(__file__): exit(2) progname = common.progname() debuglog = True logfile1 = common.tmpdir() + "/ccore1.log" logfile2 = common.tmpdir() + "/ccore2.log" indb = common.options().indb if not indb: indb = common.rootdir() + "/doc/OpeningClassification/eco.pgn" outdb = common.options().outdb if not outdb: outdb = common.testdir() + "/cfdb/eco.cfdb" if os.path.exists(logfile1): os.remove(logfile1) if os.path.exists(logfile2): os.remove(logfile2) if os.path.exists(outdb): os.remove(outdb) # Copy the database cmdline = common.ccore() if debuglog: cmdline += " --debuglog true" cmdline += " -l {0} -i {1} -o {2} copydb".format(logfile1, indb, outdb) if common.runccore(cmdline): common.checkLogfile(logfile1) # Create the opening tree in the output database cmdline = common.ccore() if debuglog: cmdline += " --debuglog true" cmdline += " -l {0} -i {1} -d 100 buildoptree".format(logfile2, outdb) if common.runccore(cmdline): common.checkLogfile(logfile2)
def main(): common.parser().add_option("--engine1", dest = "engine1", help = "Engine #1") common.parser().add_option("--engine2", dest = "engine2", help = "Engine #2") common.parser().add_option("--numgames", dest = "numgames", type = "int", default = 5, help = "Number of games") common.parser().add_option("--timecontrol", dest = "timecontrol", default = "30", help = "Time control") common.parser().add_option("--ecofile", dest = "ecofile", help = "The (.cfdb) database containing the ECO classification") common.parser().add_option("--logcomms", dest = "logcomms", type = "int", default = 0, help = "Log UCI comms") if not common.init(__file__): exit(2) progname = common.progname() debuglog = True logcomms = common.options().logcomms != 0 logfile = common.tmpdir() + "/ccore.log" pgnfile = common.tmpdir() + "/games.pgn" configfile = common.configfile() ecofile = common.options().ecofile if not ecofile: ecofile = common.testdir() + "/cfdb/eco.cfdb" engine1 = common.options().engine1 if not engine1: engine1 = common.engine1() engine2 = common.options().engine2 if not engine2: engine2 = common.engine2() numgames = common.options().numgames timecontrol = common.options().timecontrol if os.path.exists(logfile): os.remove(logfile) cmdline = common.ccore() if debuglog: cmdline += " --debuglog true" if logcomms: cmdline += " --logcomms true" cmdline += " -c {0} -l {1} -o {2} -E {3} -n {4} -t {5} tournament {6} {7}".format(configfile, logfile, pgnfile, ecofile, numgames, timecontrol, engine1, engine2) if common.runccore(cmdline): common.checkLogfile(logfile)
def r_post(m): client = init() post = client.get_post(m, wanted=wanted + ("width", "height", "imgdate", "ext", "rotate",), separate_implied=True) if not post: abort(404) data = globaldata() data.post = post data.q = "" data.extra_script = u"resize.js" data.tags = sorted(taglist(post, False) + taglist(post, True)) data.rel_posts = [Post(md5=md5) for md5 in client.post_rels(m) or []] data.rels = [] if post.rotate > 0: spec = u'%(width)dx%(height)d-%(rotate)d' % post data.svg = data.base + u'rotate/' + spec + u'/' + m + u'.' + post.ext data.ordered_tags = [t for t in post.tags if t.ordered] if data.ordered_tags: do_rel = (len(data.ordered_tags) == 1) for t in data.ordered_tags: posts = client.search_post(guids=[t.guid], order="group") pos = [p.md5 for p in posts].index(m) odata = [(u'dist2', None), (u'dist1', u'prev'), (u'dist0', None), (u'dist1', u'next'), (u'dist2', None)] start, end = pos - 2, pos + 3 if start < 0: odata = odata[-start:] start = 0 t.relposts = posts[start:end] for p, d in zip(t.relposts, odata): p.reldist = d[0] if do_rel and d[1]: data.rels.append((d[1], p.md5)) if 'aaaaaa-aaaadt-faketg-gpspos' in post.datatags: data.gps = post.datatags['aaaaaa-aaaadt-faketg-gpspos'].value return data
def r_post_tag(): client = init() m = request.forms.post post = client.get_post(m) tags = request.forms.tags create = [a.decode("utf-8") for a in request.forms.getall("create")] ctype = [a.decode("utf-8") for a in request.forms.getall("ctype")] full = set() weak = set() remove = set() failed = [] for n, t in zip(create, ctype): if t: client.add_tag(tag_clean(n), t) tags += u' ' + n for t in tags.split(): tag = client.find_tag(tag_clean(t)) if tag: p = tag_prefix(t) if p == "~": weak.add(tag) elif p == "-": remove.add(tag) else: full.add(tag) else: failed.append(t) tag_post(post, full, weak, remove) if not failed: redirect("post/" + m) data = globaldata() data.tagtypes = tagtypes() data.failed = failed data.m = m return data
def r_image(m, ext): client = init() return serve(client.image_path(m), ext)
dbName = os.path.basename(workDB).split('.')[0] #No '.' or '/' in databasenames mDBname = os.path.basename(matchDB).split('.')[0] #KOLLA imports from matchUtils import * from dbUtils import getFamilyFromId from utils import matchFam, setFamOK, setEjOKfamily, setOKperson from matchtext import matchtext from luceneUtils import setupDir, search mt_tmp = matchtext() t0 = time.time() logging.info('using db %s matching against %s', dbName, mDBname) config = common.init(dbName, matchDBName=mDBname, indexes=True) if featureSet: config['featureSet'] = featureSet if famfeatureSet: config['famfeatureSet'] = famfeatureSet common.config = config setupDir(mDBname) person_list = config['persons'] fam_list = config['families'] matches = config['matches'] matches.drop() fam_matches = config['fam_matches'] fam_matches.drop()
# -*- coding: utf-8 -*- """ Created on Sat May 18 20:45:29 2013 @author: ted """ import common as cm cm.init(centroids=[4,8,32,32,64,32,4], video_file="color_4.avi", #moving_square.avi learn_rate=0.05) cm.video_source.enableDisplayWindow() cm.network.setIsPOSTraining(True) def callback(iter): cm.printStats() #cm.network.printBeliefGraph(cm.top_layer, 0, 0) print "iter:",iter cm.printFPS() def showTree(index): if index >= tm.getMinedTreeCount(): print "out of bounds" return tm.displayMinedTree(index) tm.printMinedTreeStructure(index) cm.wk() def doTracking(delay=0):
def thumb(m, z): client = init() if z in ("normal", "large"): return serve(client.pngthumb_path(m, z), "png") else: return serve(client.thumb_path(m, z), "jpeg")
def ui_init(): common.init()
# -*- coding: utf-8 -*- # Import import common common.init(['/klass']) if __name__ == '__main__': import Template import _Selenium import CustomSelenium # Docs - common.py print common.__doc__ print ' ' + common.log.__doc__ print ' ' + common.init.__doc__ print '\n' # Docs - klass/Template.py print Template.__doc__ print ' ' + Template.TemplateKlass.__doc__ print ' ' + Template.TemplateKlass.getName.__doc__ print ' ' + Template.TemplateKlass.setName.__doc__ print '\n' # Docs - klass/_Selenium.py print _Selenium.__doc__ print ' ' + _Selenium.SeleniumKlass.__doc__ print ' ' + _Selenium.SeleniumKlass.getScreenShot.__doc__
gvFil.write( "}\n" ) gvFil.close() os.system('dot -Tsvg -O '+filnamn) #print ' Img', filnamn fil = open(filnamn+'.svg' , 'rb') graph = fil.read() fil.close() return graph if __name__=="__main__": import codecs, locale locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') #sorting?? sys.stdout = codecs.getwriter('UTF-8')(sys.stdout) import common DB = 'anders_DavidEkedahlMaster' conf = common.init(DB, matchDBName = DB, indexes=True) personDB = conf['persons'] familyDB = conf['families'] relationDB = conf['relations'] origDB = conf['originalData'] (childErr, famErr, relErr) = sanity(personDB, familyDB, relationDB) print 'Only child in one family', len(childErr) notFixed = repairChild(childErr, personDB, familyDB, relationDB, origDB) print 'Not fixed', len(notFixed) print 'Multi husb/wife in one family' repairFam(famErr, personDB, familyDB, relationDB, origDB) print 'Rel err' repairRel(relErr, personDB, familyDB, relationDB, origDB)
#!/usr/bin/env python from twisted.internet import reactor # Setup twisted logging from twisted.python import log import sys log.startLogging(sys.stdout) # Start up the common part import common common.init(reactor) # CherryPy-in-Twisted setup import cherrypy cherrypy.config.update({ 'environment': 'embedded', 'log.screen': True }) # We need to unsubscribe the CherryPy server to prevent a port conflict cherrypy.server.unsubscribe() # Start CherryPy internals cherrypy.engine.start() # Make sure we shut down CherryPy reactor.addSystemEventTrigger('after', 'shutdown', cherrypy.engine.exit) # The RFB proxy
sys.exit() #use username and first part of filename as databasename dbName = user + '_' + os.path.basename(fn).split('.')[0] logging.info('Using database %s importing from file %s', dbName, fn) #Read mappings import json (fndir,tmp) = os.path.split(fn) namMap = json.load(open(fndir + '/name.dat')) placMap = json.load(open(fndir + '/plac.dat')) datMap = json.load(open(fndir + '/date.dat')) sourMap = json.load(open(fndir + '/sour.dat')) ## config = common.init(dbName, dropWorkDB=True, indexes=True) persons = config['persons'] families = config['families'] t0 = time.time() logging.info('Reading and parsing gedcom') try: people = Gedcom(fn) except Exception, e: logging.error('<h1>Fatalt fel vid import av Gedcom</h1>') exc_type, exc_value, exc_traceback = sys.exc_info() traceback.print_exception(exc_type, exc_value, exc_traceback) sys.exit() logging.info('Time %s',time.time() - t0)
if args.json: try: result, info = do_query() print json.dumps({ 'status' : 'ok', 'result' : result }) except pymoira.BaseError as e: print json.dumps({ 'status' : 'error', 'message' : str(e) }) else: try: # Those are not real queries, and QueryInfo() would fail for them if args.query == '_help': show_help() return if args.query == '_list_queries': show_queries_list() return if args.query == '_list_users': show_user_list() return result, info = do_query() for row in result: fields = [ (field_name, row[field_name]) for field_name in info.outputs ] common.show_fields(*fields) print "" except pymoira.BaseError as err: common.error(err) if __name__ == '__main__': client, args = common.init('mrquery', 'Send raw queries to Moira', setup_arguments) handle_query()
('Alternate email', user.alternate_email) if user.alternate_email else None, ('Alternate phone', user.alternate_phone) if user.alternate_phone else None, ('Created', "%s by %s" % (common.last_modified_date(user.created_date), user.created_by)), ('Last modified', "%s by %s using %s" % (common.last_modified_date(user.lastmod_datetime), user.lastmod_by, user.lastmod_with)), ) def show_ownerships(): """Handle 'mruser ownerships'.""" user = User(client, args.user) ownership.show_ownerships(client, args, user) def setup_subcommands(argparser): """Sets up all the subcommands.""" subparsers = argparser.add_subparsers() parser_info = subparsers.add_parser('info', help = 'Provide the information about the user') parser_info.add_argument('user', help = 'The user to inspect') parser_ownerships = subparsers.add_parser('ownerships', help = 'Show items which this user owns') parser_ownerships.add_argument('user', help = 'The name of the user to show information about') parser_ownerships.add_argument('-r', '--recursive', action = 'store_true', help = 'Show items which this user own through being in lists') parser_info.set_defaults(handler = show_info) parser_ownerships.set_defaults(handler = show_ownerships) if __name__ == '__main__': client, args = common.init('mruser', 'Inspect Moira users', setup_subcommands) common.main()
root = Tk() common.error("pygtk is not available. Please install pygtk on your system.") SimpleDialog(root, i18n.get_text(common.get_user_lang(), 103) + "\n" + i18n.get_text(common.get_user_lang(), 104), ["OK"], 0, title="Omnitux").go() exit() common.info("Starting Omnitux") globalvars = common.init() common.info("Omnitux release = "+globalvars.release) common.info("Version = "+constants.version) common.info("Showing screen") pygame.display.flip() # activities and other stuff are imported here and not at the beginning # the idea is to show a black screen as soon as possible so that # the user won't click again on the omnitux icons_amount # (especially for slow systems)
# -*- coding: utf-8 -*- # This Python file uses the following encoding: utf-8 """ export a database as Gedcom """ import argparse, sys, os, datetime, re from collections import defaultdict import pickle parser = argparse.ArgumentParser() parser.add_argument("workDB", help="Working database name" ) args = parser.parse_args() workDB = args.workDB dbName = os.path.basename(workDB).split('.')[0] #No '.' or '/' in databasenames import common config = common.init(dbName, indexes=True) from common import RGDadm import codecs, locale locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') #sorting?? sys.stdout = codecs.getwriter('UTF-8')(sys.stdout) import json from workFlow import workFlowUI from dbUtils import getFamilyFromId user = workDB.split('_')[0] (files, dbs, workingDir, activeUser) = workFlowUI(user, None) sourMap = {} cIdMap = {} for rec in config['originalData'].find({'type': 'admin'}): if 'cId' in rec:
""" Created on Sat May 18 20:45:29 2013 @author: ted """ import common as cm import pydestin as pd # load the layer_widths and centroids from the # object_tracking_config.py from object_tracking_config import * isTraining = True # train from scratch, or reload from previous run cm.init(centroids=centroids, video_file="moving_square.avi", learn_rate=0.05, layer_widths=layer_widths, img_width=256) # cm.video_source.enableDisplayWindow() cm.network.setIsPOSTraining(True) write_video = True video_writer = pd.VideoWriter("obtracking_output.avi", 15) callback_iters = 0 def callback(iter): # cm.network.printBeliefGraph(cm.top_layer, 0, 0)