Example #1
0
def print_failures(prefix, outfile=sys.stdout):
    params = load_params(prefix)
    failures = []
    for level in range(1, params.search_depth + 1):
        ok_counts = collections.defaultdict(int)
        fail_counts = collections.defaultdict(int)
        for expt_name in all_experiment_names(prefix):
            for _, structure in storage.load(
                    experiments.structures_file(expt_name, level)):
                for split_id in range(params.num_splits):
                    for sample_id in range(params.num_samples):
                        ok = False
                        fname = experiments.scores_file(
                            expt_name, level, structure, split_id, sample_id)
                        if storage.exists(fname):
                            row_loglik, col_loglik = storage.load(fname)
                            if np.all(np.isfinite(row_loglik)) and np.all(
                                    np.isfinite(col_loglik)):
                                ok = True

                        if ok:
                            ok_counts[structure] += 1
                        else:
                            fail_counts[structure] += 1

        for structure in fail_counts:
            if ok_counts[structure] > 0:
                failures.append(presentation.Failure(structure, level, False))
            else:
                failures.append(presentation.Failure(structure, level, True))

    presentation.print_failed_structures(failures, outfile)
def print_failures(prefix, outfile=sys.stdout):
    params = load_params(prefix)
    failures = []
    for level in range(1, params.search_depth + 1):
        ok_counts = collections.defaultdict(int)
        fail_counts = collections.defaultdict(int)
        for expt_name in all_experiment_names(prefix):
            for _, structure in storage.load(experiments.structures_file(expt_name, level)):
                for split_id in range(params.num_splits):
                    for sample_id in range(params.num_samples):
                        ok = False
                        fname = experiments.scores_file(expt_name, level, structure, split_id, sample_id)
                        if storage.exists(fname):
                            row_loglik, col_loglik = storage.load(fname)
                            if np.all(np.isfinite(row_loglik)) and np.all(np.isfinite(col_loglik)):
                                ok = True

                        if ok:
                            ok_counts[structure] += 1
                        else:
                            fail_counts[structure] += 1

        for structure in fail_counts:
            if ok_counts[structure] > 0:
                failures.append(presentation.Failure(structure, level, False))
            else:
                failures.append(presentation.Failure(structure, level, True))

    presentation.print_failed_structures(failures, outfile)
Example #3
0
def init_level(name, level):
    """Initialize a given level of the search by saving all of the structures which need
    to be evaluated."""
    if not storage.exists(experiment_dir(name)):
        raise RuntimeError('Experiment %s not yet initialized.' % name)

    params = storage.load(params_file(name))
    splits = storage.load(splits_file(name))

    if level == 1:
        init_structures = ['g']
    else:
        init_structures = storage.load(winning_structure_file(name, level - 1))

    structure_pairs = list_structure_pairs(init_structures, params.rules, params.expand_noise)
    data_matrix = storage.load(data_file(name))
    X_train = data_matrix
    lab = None
    node_mat = np.zeros([params.num_splits * params.num_samples, 200, 200, 2])
    pruned_pairs = []
    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True  # ; config.gpu_options.per_process_gpu_memory_fraction = 0.4
    sess = tf.Session(config=config)
    c = Classifier()
    real = tf.placeholder(shape=[None, 200, 200, 2], dtype=tf.float32)
    c_out = tf.reduce_mean(tf.nn.softmax(c(real), axis=-1), axis=0, keepdims=True)
    c_params = c.vars
    saver = tf.train.Saver(c_params)
    sess.run(tf.global_variables_initializer())
    saver.restore(sess, "saved_model/d")
    for (init_structure, structure) in structure_pairs:
        for split_id in range(params.num_splits):
            for sample_id in range(params.num_samples):
                train_rows, train_cols, test_rows, test_cols = splits[split_id]
                X_train = data_matrix[train_rows[:, nax], train_cols[nax, :]]
                if level == 1:
                    init = X_train.sample_latent_values(np.zeros((X_train.m, X_train.n)), 1.)
                    prev_model = recursive.GaussianNode(init, 'scalar', 1.)
                else:
                    try:
                        prev_model = storage.load(samples_file(name, level - 1, init_structure, split_id, sample_id))
                    except:
                        print("structure", grammar.pretty_print(init_structure), "never exists")
                        continue
                    if isinstance(prev_model, recursive.Decomp):
                        prev_model = prev_model.root
                node, old_dist, rule = recursive.find_changed_node(prev_model, init_structure, structure)
                lab = labelize(rule)
                node_mat[split_id * params.num_samples + sample_id] = pad(random_shrink(node.value()))

        if_continue = sess.run(tf.nn.top_k(c_out, 3), feed_dict={real: node_mat})
        if lab in if_continue.indices:
            print("transformation structure ", grammar.pretty_print(init_structure), "->", grammar.pretty_print(structure), "i.e. lab ", lab,
                  " included with top_k", if_continue)
            pruned_pairs.append((init_structure, structure))
        else:
            print("transformation structure ", grammar.pretty_print(init_structure), "->", grammar.pretty_print(structure), "i.e. lab ", lab, " emitted, with top_k", if_continue)
    structure_pairs = pruned_pairs
    storage.dump(structure_pairs, structures_file(name, level))
def check_required_directories():
    config_vars = ['RESULTS_PATH', 'JOBS_PATH', 'REPORT_PATH']
    for v in config_vars:
        if not hasattr(config, v):
            raise RuntimeError('Need to specify %s in config.py' % v)
        if not storage.exists(getattr(config, v)):
            raise RuntimeError('Directory specified in config.%s does not exist: %s' %
                               (v, getattr(config, v)))
Example #5
0
def check_required_directories():
    config_vars = ['RESULTS_PATH', 'JOBS_PATH', 'REPORT_PATH']
    for v in config_vars:
        if not hasattr(config, v):
            raise RuntimeError('Need to specify %s in config.py' % v)
        if not storage.exists(getattr(config, v)):
            raise RuntimeError('Directory specified in config.%s does not exist: %s' %
                               (v, getattr(config, v)))
def update_musicscores():
    logger.debug("Updating all musicscores")
    if not storage.exists(MANIFEST_PATH):
        logger.debug("manifest.db not found, updating metadata")
        meta_updater.update_database()
    with db.CustomDB(meta_updater.get_manifestdb_path()) as manifest_conn:
        all_musicscores = manifest_conn.execute_and_fetchall("""
            SELECT name,hash FROM manifests WHERE (name LIKE "musicscores\_m___.bdb" ESCAPE '\\')
            """)
        all_musicscores = {_[0].split(".")[0]: _[1] for _ in all_musicscores}

    if not _score_cache_db_exists():
        _initialize_score_cache_db()
        new_scores = all_musicscores.keys()
        updated_scores = set()
    else:
        scores_meta = db.cachedb.execute_and_fetchall(
            "SELECT score_id, score_hash FROM score_cache")
        scores_meta = {_: __ for _, __ in scores_meta}
        deleted_scores = set(scores_meta.keys()).difference(
            all_musicscores.keys())
        if len(deleted_scores) > 0:
            logger.info(
                "Found {} defunct musicscores, removing them...".format(
                    len(deleted_scores)))
            for deleted_score in deleted_scores:
                path = MUSICSCORES_PATH / "{}.db".format(deleted_score)
                path.unlink()
        new_scores = set(all_musicscores.keys()).difference(scores_meta.keys())
        updated_scores = [
            _ for _ in set(all_musicscores.keys()).intersection(
                scores_meta.keys()) if scores_meta[_] != all_musicscores[_]
        ]
    logger.info(
        "Found {} musicscores, {} of them are new, {} are updated...".format(
            len(all_musicscores), len(new_scores), len(updated_scores)))

    if len(new_scores) + len(updated_scores) > 50:
        logger.info("It will take some time to download, please wait...")

    for musicscore_name in set(new_scores).union(set(updated_scores)):
        musicscore_hash = all_musicscores[musicscore_name]
        musicscore_response = cgss_query.get_db(musicscore_hash)
        with storage.get_writer(
                MUSICSCORES_PATH / "{}.db".format(musicscore_name),
                'wb') as fwb:
            fwb.write(decompress(musicscore_response.content))
        db.cachedb.execute(
            """
            INSERT OR REPLACE INTO score_cache (score_id, score_hash)
            VALUES (?,?)
        """, [musicscore_name, musicscore_hash])
    db.cachedb.commit()
    logger.info("All musicscores updated")
def init_level(name, level):
    """Initialize a given level of the search by saving all of the structures which need
    to be evaluated."""
    if not storage.exists(experiment_dir(name)):
        raise RuntimeError('Experiment %s not yet initialized.' % name)

    params = storage.load(params_file(name))
    if level == 1:
        init_structures = ['g']
    else:
        init_structures = storage.load(winning_structure_file(name, level - 1))
    structure_pairs = list_structure_pairs(init_structures, params.rules, params.expand_noise)
    storage.dump(structure_pairs, structures_file(name, level))
Example #8
0
def init_level(name, level):
    """Initialize a given level of the search by saving all of the structures which need
    to be evaluated."""
    if not storage.exists(experiment_dir(name)):
        raise RuntimeError('Experiment %s not yet initialized.' % name)

    params = storage.load(params_file(name))
    if level == 1:
        init_structures = ['g']
    else:
        init_structures = storage.load(winning_structure_file(name, level - 1))
    structure_pairs = list_structure_pairs(init_structures, params.rules,
                                           params.expand_noise)
    storage.dump(structure_pairs, structures_file(name, level))
def setup(update=False):
    load_static()
    if not (storage.exists(MANIFEST_PATH) and storage.exists(MASTERDB_PATH)):
        update = True
    from db import db
    assert db
    from network import meta_updater
    if update:
        meta_updater.update_database()
    from logic.search import card_query
    assert card_query
    from network import music_updater
    music_updater.update_musicscores()
    from network import chart_cache_updater
    chart_cache_updater.update_cache_scores()
    from network import image_updater
    assert image_updater
    from logic.profile import profile_manager
    assert profile_manager
    from logic.search import indexer, search_engine
    assert indexer
    assert search_engine
    from gui.viewmodels.chart_viewer import listener
    assert listener
def update_image(card_id, sleep=0.1):
    path = IMAGE_PATH / "{:06d}.png".format(card_id)
    path32 = IMAGE_PATH32 / "{:06d}.jpg".format(card_id)
    path64 = IMAGE_PATH64 / "{:06d}.jpg".format(card_id)
    if not storage.exists(path):
        time.sleep(sleep)
        r = requests.get(FORMAT.format(card_id))
        if r.status_code == 200:
            with storage.get_writer(path, 'wb') as fwb:
                for chunk in r:
                    fwb.write(chunk)
    if not path64.exists() and not path32.exists():
        img = Image.open(str(path)).convert('RGB')
        if not path32.exists():
            img.resize((32, 32), Image.ANTIALIAS).save(str(path32), format='JPEG')
        if not path64.exists():
            img.resize((64, 64), Image.ANTIALIAS).save(str(path64), format='JPEG')
Example #11
0
def init_experiment(name, data_matrix, params, components=None, clean_data_matrix=None):
    """Initialize the structure search by saving the matrix, and possibly auxiliary
    information, to files, and generating cross-validation splits."""
    check_required_directories()

    if not storage.exists(experiment_dir(name)):
        storage.mkdir(experiment_dir(name))

    storage.dump(params, params_file(name))
    splits = nfold_cv(data_matrix.m, data_matrix.n, params.num_splits)
    storage.dump(splits, splits_file(name))

    if clean_data_matrix is not None:
        storage.dump(clean_data_matrix, clean_data_file(name))

    storage.dump(data_matrix, data_file(name))

    if components is not None:
        storage.dump(components, components_file(name))
def init_experiment(name, data_matrix, params, components=None, clean_data_matrix=None):
    """Initialize the structure search by saving the matrix, and possibly auxiliary
    information, to files, and generating cross-validation splits."""
    check_required_directories()
    
    if not storage.exists(experiment_dir(name)):
        storage.mkdir(experiment_dir(name))

    storage.dump(params, params_file(name))
    splits = nfold_cv(data_matrix.m, data_matrix.n, params.num_splits)
    storage.dump(splits, splits_file(name))

    if clean_data_matrix is not None:
        storage.dump(clean_data_matrix, clean_data_file(name))

    storage.dump(data_matrix, data_file(name))

    if components is not None:
        storage.dump(components, components_file(name))
Example #13
0
def print_failures(name, outfile=sys.stdout):
    params = storage.load(params_file(name))
    failures = []
    for level in range(1, params.search_depth + 1):
        for _, structure in storage.load(structures_file(name, level)):
            total_ok = 0

            for split_id in range(params.num_splits):
                for sample_id in range(params.num_samples):
                    fname = scores_file(name, level, structure, split_id, sample_id)
                    if storage.exists(fname):
                        row_loglik, col_loglik = storage.load(fname)
                        if np.all(np.isfinite(row_loglik)) and np.all(np.isfinite(col_loglik)):
                            total_ok += 1

            if total_ok == 0:
                failures.append(presentation.Failure(structure, level, True))
            elif total_ok < params.num_splits * params.num_samples:
                failures.append(presentation.Failure(structure, level, False))

    presentation.print_failed_structures(failures, outfile)
def print_failures(name, outfile=sys.stdout):
    params = storage.load(params_file(name))
    failures = []
    for level in range(1, params.search_depth + 1):
        for _, structure in storage.load(structures_file(name, level)):
            total_ok = 0
            
            for split_id in range(params.num_splits):
                for sample_id in range(params.num_samples):
                    fname = scores_file(name, level, structure, split_id, sample_id)
                    if storage.exists(fname):
                        row_loglik, col_loglik = storage.load(fname)
                        if np.all(np.isfinite(row_loglik)) and np.all(np.isfinite(col_loglik)):
                            total_ok += 1

            if total_ok == 0:
                failures.append(presentation.Failure(structure, level, True))
            elif total_ok < params.num_splits * params.num_samples:
                failures.append(presentation.Failure(structure, level, False))

    presentation.print_failed_structures(failures, outfile)
def get_cachedb_path():
    if not storage.exists(CACHEDB_PATH):
        conn = sqlite3.connect(CACHEDB_PATH)
        conn.close()
    return CACHEDB_PATH
def get_masterdb_path():
    if not storage.exists(MASTERDB_PATH):
        logger.debug("master.db not found, triggering manifest updater")
        _update_masterdb()
    return MASTERDB_PATH
def get_manifestdb_path():
    if not storage.exists(MANIFEST_PATH):
        logger.debug("manifest.db not found, triggering manifest updater")
        _update_manifest()
    return MANIFEST_PATH