def main(): # Args parser parser = argparse.ArgumentParser(description= "Evaluates the average rank and mean AP for the test SHS " \ "over the entire MSD", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("msd_dir", action="store", help="Million Song Dataset main directory") parser.add_argument("-dictfile", action="store", default="", help="Pickle to the learned dictionary") parser.add_argument("-outdir", action="store", default="msd_codes", help="Output directory for the features") parser.add_argument("-N", action="store", default=10, type=int, help="Number of processors to use when computing " \ "the codes for 1M tracks,") parser.add_argument("-lda", action="store", default=None, help="LDA file") parser.add_argument("-pca", nargs=2, metavar=('f.pkl', 'n'), default=(None, 0), help="pca model saved in a pickle file, " \ "use n dimensions") parser.add_argument("-codes", action="store", nargs=2, default=[None,0], dest="codesdir", metavar=("msd_codes/", "n"), help="Path to the folder with all the codes and " "version to evaluate") parser.add_argument("-orig_codes", action="store", default=None, dest="origcodesdir", help="Path to the folder with all the codes without " "dimensionality reduction") parser.add_argument("-norm", action="store_true", dest="norm", default=False, help="Normalize before LDA/PCA or not") args = parser.parse_args() start_time = time.time() maindir = args.msd_dir shsf = "SHS/shs_dataset_test.txt" global lda global pca # sanity cheks utils.assert_file(maindir) utils.assert_file(shsf) utils.create_dir(args.outdir) # read cliques and all tracks cliques, all_tracks = utils.read_shs_file(shsf) track_ids = utils.load_pickle("SHS/track_ids_test.pk") clique_ids = utils.load_pickle("SHS/clique_ids_test.pk") # read codes file codesdir = args.codesdir[0] if codesdir is not None: if os.path.isfile(codesdir): c = utils.load_pickle(codesdir) feats = c[0] track_ids = c[1] clique_ids = c[2] else: feats, track_ids, clique_ids = load_codes(codesdir, lda_idx=int(args.codesdir[1])) logger.info("Codes files read") print feats.shape else: # Read PCA file if args.pca[0] is not None: pca = utils.load_pickle(args.pca[0])[int(args.pca[1])] # read LDA file lda_file = args.lda if lda_file is not None: lda = utils.load_pickle(lda_file) utils.assert_file(args.dictfile) # Prepare Multiprocessing computation input = [] pool = Pool(processes=args.N) for n in xrange(args.N): arg = {} arg["track_ids"] = track_ids arg["maindir"] = maindir arg["d"] = args.dictfile arg["N"] = n arg["clique_ids"] = clique_ids arg["outdir"] = args.outdir arg["origcodesdir"] = args.origcodesdir arg["pca_n"] = int(args.pca[1]) arg["norm"] = args.norm input.append(arg) # Start computing the codes pool.map(compute_codes, input) # Done! logger.info("Codes computation done!") logger.info("Took %.2f seconds" % (time.time() - start_time)) sys.exit() # Scores feats, clique_ids, track_ids = utils.clean_feats(feats, clique_ids, track_ids) stats = score(feats, clique_ids, N=len(all_tracks)) # TODO: change file name utils.save_pickle(stats, "stats.pk") # done logger.info('Average rank per track: %.2f, clique: %.2f, MAP: %.2f%%' \ % (anst.average_rank_per_track(stats), anst.average_rank_per_clique(stats), anst.mean_average_precision(stats) * 100)) logger.info("Done! Took %.2f seconds" % (time.time() - start_time))
def test_kubernetes_manifest_file(host): assert_file(host, "/etc/kubernetes/manifests/keepalived.yml", "root", "root") assert_file(host, "/etc/kubernetes/manifests/haproxy.yml", "root", "root")
def main(): # Args parser parser = argparse.ArgumentParser( description="Evaluates the 500 binary queries from the SHS data set", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("msd_dir", action="store", help="Million Song Dataset main directory") parser.add_argument("-dictfile", action="store", default="", help="Pickle to the learned dictionary") parser.add_argument("-lda", action="store", nargs=2, default=[None, 0], help="LDA file and version", metavar=('lda.pkl', 'n')) parser.add_argument("-pca", nargs=2, metavar=('f.pkl', 'n'), default=("", 0), help="pca model saved in a pickle file, " \ "use n dimensions") # Parse args = parser.parse_args() # Track time start_time = time.time() maindir = args.msd_dir queriesf = "SHS/list_500queries.txt" shsf = "SHS/shs_dataset_train.txt" lda = args.lda[0] lda_n = int(args.lda[1]) pcafile = args.pca[0] pcadim = int(args.pca[1]) # sanity cheks utils.assert_file(maindir) utils.assert_file(queriesf) utils.assert_file(shsf) utils.assert_file(pcafile) # read queries queries = read_query_file(queriesf) # load pca trainedpca = None if pcafile != "": f = open(pcafile, 'r') trainedpca = cPickle.load(f) f.close() assert pcadim > 0 logger.info('trained pca loaded') # load lda if lda != None: lda = utils.load_pickle(lda) # to keep stats results = [] # iterate over queries logger.info("Starting the binary task...") # Get the dictionary transform td = load_transform(args.dictfile) for triplet in queries: # get features filenames = map(lambda tid: utils.path_from_tid(maindir, tid), triplet) triplet_feats = map( lambda f: extract_feats(f, td=td, lda_file=lda, lda_n=lda_n), filenames) if None in triplet_feats: continue # Apply pca if needed if trainedpca: triplet_feats = map(lambda feat: \ trainedpca.apply_newdata(feat, ndims=pcadim), triplet_feats) assert triplet_feats[np.random.randint(3)].shape[0] == pcadim # Compute result res1 = triplet_feats[0] - triplet_feats[1] res1 = np.sum(res1 * res1) res2 = triplet_feats[0] - triplet_feats[2] res2 = np.sum(res2 * res2) if res1 < res2: results.append(1) else: results.append(0) # verbose if len(results) % 5 == 0: logger.info(' --- after %d queries, accuracy: %.1f %%' % \ (len(results), 100. * np.mean(results))) # done logger.info('After %d queries, accuracy: %.1f %%' % (len(results), 100. * np.mean(results))) logger.info('Done! Took %.2f seconds' % (time.time() - start_time))
def test_keepalived_conf_files(host): assert_file(host, "/etc/keepalived/keepalived.conf", "root", "root") assert_file(host, "/etc/keepalived/check_apiserver.sh", "root", "root")
def test_haproxy_conf_file(host): assert_file(host, "/etc/haproxy/haproxy.cfg", "root", "root")
def main(): # Args parser parser = argparse.ArgumentParser(description= "Cover song ID on the training Second Hand Song dataset", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("msd_dir", action="store", help="Million Song Dataset main directory") parser.add_argument("-dictfile", action="store", default="", help="Pickle to the learned dictionary") parser.add_argument("-lda", action="store", nargs=2, default=[None,0], help="LDA file and version", metavar=('lda.pkl', 'n')) parser.add_argument("-codes", action="store", default=None, dest="codesfile", help="Pickle to the features file") parser.add_argument("-f", action="store", default="", dest="featfile", help="Pickle to the final features") parser.add_argument("-pca", nargs=2, metavar=('f.pkl', 'n'), default=("", 0), help="pca model saved in a pickle file, " \ "use n dimensions") args = parser.parse_args() start_time = time.time() maindir = args.msd_dir shsf = "SHS/shs_dataset_train.txt" dictfile = args.dictfile # sanity cheks utils.assert_file(dictfile) utils.assert_file(maindir) utils.assert_file(shsf) # read clique ids and track ids cliques, all_tracks = utils.read_shs_file(shsf) track_ids = all_tracks.keys() clique_ids = np.asarray(utils.compute_clique_idxs(track_ids, cliques)) logger.info("Track ids and clique ids read") utils.save_pickle(clique_ids, "SHS/clique_ids_train.pk") utils.save_pickle(track_ids, "SHS/track_ids_train.pk") # read LDA file lda_file = args.lda[0] if lda_file != None: lda_file = utils.load_pickle(lda_file) logger.info("LDA file read") # read codes file codesfile = args.codesfile if codesfile != None: codesfile = utils.load_pickle(codesfile) logger.info("Codes file read") # Compute features if needed if args.featfile == "": feats = compute_feats(track_ids, maindir, dictfile, lda_file=lda_file, lda_n=int(args.lda[1]), codes=codesfile, pca=args.pca[0], pca_n=int(args.pca[1])) else: feats = utils.load_pickle(args.featfile) # Apply PCA pcafile = args.pca[0] pcadim = int(args.pca[1]) if pcafile != "" and False: trainedpca = utils.load_pickle(pcafile) assert pcadim > 0 logger.info('trained pca loaded') pcafeats = np.zeros((feats.shape[0], pcadim)) for i,feat in enumerate(feats): pcafeats[i] = trainedpca.apply_newdata(feat, ndims=pcadim) feats = pcafeats # Scores feats, clique_ids, track_ids = utils.clean_feats(feats, clique_ids, track_ids) stats = score(feats, clique_ids) # Save data if dictfile == "": dictfile = "thierry" # For saving purposes utils.save_pickle(stats, "results/stats-" + os.path.basename(dictfile) + ".pk") # done logger.info('Average rank per track: %.2f, clique: %.2f, MAP: %.2f%%' \ % (anst.average_rank_per_track(stats), anst.average_rank_per_clique(stats), anst.mean_average_precision(stats) * 100)) logger.info("Done! Took %.2f seconds" % (time.time() - start_time))
def main(): # Args parser parser = argparse.ArgumentParser( description="Cover song ID on the training Second Hand Song dataset", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("msd_dir", action="store", help="Million Song Dataset main directory") parser.add_argument("-dictfile", action="store", default="", help="Pickle to the learned dictionary") parser.add_argument("-lda", action="store", nargs=2, default=[None, 0], help="LDA file and version", metavar=('lda.pkl', 'n')) parser.add_argument("-codes", action="store", default=None, dest="codesfile", help="Pickle to the features file") parser.add_argument("-f", action="store", default="", dest="featfile", help="Pickle to the final features") parser.add_argument("-pca", nargs=2, metavar=('f.pkl', 'n'), default=("", 0), help="pca model saved in a pickle file, " \ "use n dimensions") args = parser.parse_args() start_time = time.time() maindir = args.msd_dir shsf = "SHS/shs_dataset_train.txt" dictfile = args.dictfile # sanity cheks utils.assert_file(dictfile) utils.assert_file(maindir) utils.assert_file(shsf) # read clique ids and track ids cliques, all_tracks = utils.read_shs_file(shsf) track_ids = all_tracks.keys() clique_ids = np.asarray(utils.compute_clique_idxs(track_ids, cliques)) logger.info("Track ids and clique ids read") utils.save_pickle(clique_ids, "SHS/clique_ids_train.pk") utils.save_pickle(track_ids, "SHS/track_ids_train.pk") # read LDA file lda_file = args.lda[0] if lda_file != None: lda_file = utils.load_pickle(lda_file) logger.info("LDA file read") # read codes file codesfile = args.codesfile if codesfile != None: codesfile = utils.load_pickle(codesfile) logger.info("Codes file read") # Compute features if needed if args.featfile == "": feats = compute_feats(track_ids, maindir, dictfile, lda_file=lda_file, lda_n=int(args.lda[1]), codes=codesfile, pca=args.pca[0], pca_n=int(args.pca[1])) else: feats = utils.load_pickle(args.featfile) # Apply PCA pcafile = args.pca[0] pcadim = int(args.pca[1]) if pcafile != "" and False: trainedpca = utils.load_pickle(pcafile) assert pcadim > 0 logger.info('trained pca loaded') pcafeats = np.zeros((feats.shape[0], pcadim)) for i, feat in enumerate(feats): pcafeats[i] = trainedpca.apply_newdata(feat, ndims=pcadim) feats = pcafeats # Scores feats, clique_ids, track_ids = utils.clean_feats(feats, clique_ids, track_ids) stats = score(feats, clique_ids) # Save data if dictfile == "": dictfile = "thierry" # For saving purposes utils.save_pickle(stats, "results/stats-" + os.path.basename(dictfile) + ".pk") # done logger.info('Average rank per track: %.2f, clique: %.2f, MAP: %.2f%%' \ % (anst.average_rank_per_track(stats), anst.average_rank_per_clique(stats), anst.mean_average_precision(stats) * 100)) logger.info("Done! Took %.2f seconds" % (time.time() - start_time))
def main(): # Args parser parser = argparse.ArgumentParser(description= "Evaluates the average rank and mean AP for the test SHS " \ "over the entire MSD", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("msd_dir", action="store", help="Million Song Dataset main directory") parser.add_argument("-dictfile", action="store", default="", help="Pickle to the learned dictionary") parser.add_argument("-outdir", action="store", default="msd_codes", help="Output directory for the features") parser.add_argument("-N", action="store", default=10, type=int, help="Number of processors to use when computing " \ "the codes for 1M tracks,") parser.add_argument("-lda", action="store", default=None, help="LDA file") parser.add_argument("-pca", nargs=2, metavar=('f.pkl', 'n'), default=(None, 0), help="pca model saved in a pickle file, " \ "use n dimensions") parser.add_argument("-codes", action="store", nargs=2, default=[None, 0], dest="codesdir", metavar=("msd_codes/", "n"), help="Path to the folder with all the codes and " "version to evaluate") parser.add_argument("-orig_codes", action="store", default=None, dest="origcodesdir", help="Path to the folder with all the codes without " "dimensionality reduction") parser.add_argument("-norm", action="store_true", dest="norm", default=False, help="Normalize before LDA/PCA or not") args = parser.parse_args() start_time = time.time() maindir = args.msd_dir shsf = "SHS/shs_dataset_test.txt" global lda global pca # sanity cheks utils.assert_file(maindir) utils.assert_file(shsf) utils.create_dir(args.outdir) # read cliques and all tracks cliques, all_tracks = utils.read_shs_file(shsf) track_ids = utils.load_pickle("SHS/track_ids_test.pk") clique_ids = utils.load_pickle("SHS/clique_ids_test.pk") # read codes file codesdir = args.codesdir[0] if codesdir is not None: if os.path.isfile(codesdir): c = utils.load_pickle(codesdir) feats = c[0] track_ids = c[1] clique_ids = c[2] else: feats, track_ids, clique_ids = load_codes(codesdir, lda_idx=int( args.codesdir[1])) logger.info("Codes files read") print feats.shape else: # Read PCA file if args.pca[0] is not None: pca = utils.load_pickle(args.pca[0])[int(args.pca[1])] # read LDA file lda_file = args.lda if lda_file is not None: lda = utils.load_pickle(lda_file) utils.assert_file(args.dictfile) # Prepare Multiprocessing computation input = [] pool = Pool(processes=args.N) for n in xrange(args.N): arg = {} arg["track_ids"] = track_ids arg["maindir"] = maindir arg["d"] = args.dictfile arg["N"] = n arg["clique_ids"] = clique_ids arg["outdir"] = args.outdir arg["origcodesdir"] = args.origcodesdir arg["pca_n"] = int(args.pca[1]) arg["norm"] = args.norm input.append(arg) # Start computing the codes pool.map(compute_codes, input) # Done! logger.info("Codes computation done!") logger.info("Took %.2f seconds" % (time.time() - start_time)) sys.exit() # Scores feats, clique_ids, track_ids = utils.clean_feats(feats, clique_ids, track_ids) stats = score(feats, clique_ids, N=len(all_tracks)) # TODO: change file name utils.save_pickle(stats, "stats.pk") # done logger.info('Average rank per track: %.2f, clique: %.2f, MAP: %.2f%%' \ % (anst.average_rank_per_track(stats), anst.average_rank_per_clique(stats), anst.mean_average_precision(stats) * 100)) logger.info("Done! Took %.2f seconds" % (time.time() - start_time))
def main(): # Args parser parser = argparse.ArgumentParser( description="Evaluates the 500 binary queries from the SHS data set", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser.add_argument("msd_dir", action="store", help="Million Song Dataset main directory") parser.add_argument("-dictfile", action="store", default="", help="Pickle to the learned dictionary") parser.add_argument( "-lda", action="store", nargs=2, default=[None, 0], help="LDA file and version", metavar=("lda.pkl", "n") ) parser.add_argument( "-pca", nargs=2, metavar=("f.pkl", "n"), default=("", 0), help="pca model saved in a pickle file, " "use n dimensions", ) # Parse args = parser.parse_args() # Track time start_time = time.time() maindir = args.msd_dir queriesf = "SHS/list_500queries.txt" shsf = "SHS/shs_dataset_train.txt" lda = args.lda[0] lda_n = int(args.lda[1]) pcafile = args.pca[0] pcadim = int(args.pca[1]) # sanity cheks utils.assert_file(maindir) utils.assert_file(queriesf) utils.assert_file(shsf) utils.assert_file(pcafile) # read queries queries = read_query_file(queriesf) # load pca trainedpca = None if pcafile != "": f = open(pcafile, "r") trainedpca = cPickle.load(f) f.close() assert pcadim > 0 logger.info("trained pca loaded") # load lda if lda != None: lda = utils.load_pickle(lda) # to keep stats results = [] # iterate over queries logger.info("Starting the binary task...") # Get the dictionary transform td = load_transform(args.dictfile) for triplet in queries: # get features filenames = map(lambda tid: utils.path_from_tid(maindir, tid), triplet) triplet_feats = map(lambda f: extract_feats(f, td=td, lda_file=lda, lda_n=lda_n), filenames) if None in triplet_feats: continue # Apply pca if needed if trainedpca: triplet_feats = map(lambda feat: trainedpca.apply_newdata(feat, ndims=pcadim), triplet_feats) assert triplet_feats[np.random.randint(3)].shape[0] == pcadim # Compute result res1 = triplet_feats[0] - triplet_feats[1] res1 = np.sum(res1 * res1) res2 = triplet_feats[0] - triplet_feats[2] res2 = np.sum(res2 * res2) if res1 < res2: results.append(1) else: results.append(0) # verbose if len(results) % 5 == 0: logger.info(" --- after %d queries, accuracy: %.1f %%" % (len(results), 100.0 * np.mean(results))) # done logger.info("After %d queries, accuracy: %.1f %%" % (len(results), 100.0 * np.mean(results))) logger.info("Done! Took %.2f seconds" % (time.time() - start_time))
def test_crio_sysctl_d_conf_file(host): assert_file(host, "/etc/sysctl.d/99-kubernetes-cri.conf", "root", "root")
def test_crio_module_load_d_conf_file(host): assert_file(host, "/etc/modules-load.d/crio.conf", "root", "root")
def test_crio_conf_file(host): assert_file(host, "/etc/crio/crio.conf.d/02-cgroup-manager.conf", "root", "root")