def print_failures(prefix, outfile=sys.stdout):
params = load_params(prefix)
failures = []
for level in range(1, params.search_depth + 1):
ok_counts = collections.defaultdict(int)
fail_counts = collections.defaultdict(int)
for expt_name in all_experiment_names(prefix):
for _, structure in storage.load(
experiments.structures_file(expt_name, level)):
for split_id in range(params.num_splits):
for sample_id in range(params.num_samples):
ok = False
fname = experiments.scores_file(
expt_name, level, structure, split_id, sample_id)
if storage.exists(fname):
row_loglik, col_loglik = storage.load(fname)
if np.all(np.isfinite(row_loglik)) and np.all(
np.isfinite(col_loglik)):
ok = True
if ok:
ok_counts[structure] += 1
else:
fail_counts[structure] += 1
for structure in fail_counts:
if ok_counts[structure] > 0:
failures.append(presentation.Failure(structure, level, False))
else:
failures.append(presentation.Failure(structure, level, True))
presentation.print_failed_structures(failures, outfile)
def print_failures(prefix, outfile=sys.stdout):
params = load_params(prefix)
failures = []
for level in range(1, params.search_depth + 1):
ok_counts = collections.defaultdict(int)
fail_counts = collections.defaultdict(int)
for expt_name in all_experiment_names(prefix):
for _, structure in storage.load(experiments.structures_file(expt_name, level)):
for split_id in range(params.num_splits):
for sample_id in range(params.num_samples):
ok = False
fname = experiments.scores_file(expt_name, level, structure, split_id, sample_id)
if storage.exists(fname):
row_loglik, col_loglik = storage.load(fname)
if np.all(np.isfinite(row_loglik)) and np.all(np.isfinite(col_loglik)):
ok = True
if ok:
ok_counts[structure] += 1
else:
fail_counts[structure] += 1
for structure in fail_counts:
if ok_counts[structure] > 0:
failures.append(presentation.Failure(structure, level, False))
else:
failures.append(presentation.Failure(structure, level, True))
presentation.print_failed_structures(failures, outfile)
def init_level(name, level):
"""Initialize a given level of the search by saving all of the structures which need
to be evaluated."""
if not storage.exists(experiment_dir(name)):
raise RuntimeError('Experiment %s not yet initialized.' % name)
params = storage.load(params_file(name))
splits = storage.load(splits_file(name))
if level == 1:
init_structures = ['g']
else:
init_structures = storage.load(winning_structure_file(name, level - 1))
structure_pairs = list_structure_pairs(init_structures, params.rules, params.expand_noise)
data_matrix = storage.load(data_file(name))
X_train = data_matrix
lab = None
node_mat = np.zeros([params.num_splits * params.num_samples, 200, 200, 2])
pruned_pairs = []
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
config = tf.ConfigProto()
config.gpu_options.allow_growth = True # ; config.gpu_options.per_process_gpu_memory_fraction = 0.4
sess = tf.Session(config=config)
c = Classifier()
real = tf.placeholder(shape=[None, 200, 200, 2], dtype=tf.float32)
c_out = tf.reduce_mean(tf.nn.softmax(c(real), axis=-1), axis=0, keepdims=True)
c_params = c.vars
saver = tf.train.Saver(c_params)
sess.run(tf.global_variables_initializer())
saver.restore(sess, "saved_model/d")
for (init_structure, structure) in structure_pairs:
for split_id in range(params.num_splits):
for sample_id in range(params.num_samples):
train_rows, train_cols, test_rows, test_cols = splits[split_id]
X_train = data_matrix[train_rows[:, nax], train_cols[nax, :]]
if level == 1:
init = X_train.sample_latent_values(np.zeros((X_train.m, X_train.n)), 1.)
prev_model = recursive.GaussianNode(init, 'scalar', 1.)
else:
try:
prev_model = storage.load(samples_file(name, level - 1, init_structure, split_id, sample_id))
except:
print("structure", grammar.pretty_print(init_structure), "never exists")
continue
if isinstance(prev_model, recursive.Decomp):
prev_model = prev_model.root
node, old_dist, rule = recursive.find_changed_node(prev_model, init_structure, structure)
lab = labelize(rule)
node_mat[split_id * params.num_samples + sample_id] = pad(random_shrink(node.value()))
if_continue = sess.run(tf.nn.top_k(c_out, 3), feed_dict={real: node_mat})
if lab in if_continue.indices:
print("transformation structure ", grammar.pretty_print(init_structure), "->", grammar.pretty_print(structure), "i.e. lab ", lab,
" included with top_k", if_continue)
pruned_pairs.append((init_structure, structure))
else:
print("transformation structure ", grammar.pretty_print(init_structure), "->", grammar.pretty_print(structure), "i.e. lab ", lab, " emitted, with top_k", if_continue)
structure_pairs = pruned_pairs
storage.dump(structure_pairs, structures_file(name, level))
def check_required_directories():
config_vars = ['RESULTS_PATH', 'JOBS_PATH', 'REPORT_PATH']
for v in config_vars:
if not hasattr(config, v):
raise RuntimeError('Need to specify %s in config.py' % v)
if not storage.exists(getattr(config, v)):
raise RuntimeError('Directory specified in config.%s does not exist: %s' %
(v, getattr(config, v)))
def check_required_directories():
config_vars = ['RESULTS_PATH', 'JOBS_PATH', 'REPORT_PATH']
for v in config_vars:
if not hasattr(config, v):
raise RuntimeError('Need to specify %s in config.py' % v)
if not storage.exists(getattr(config, v)):
raise RuntimeError('Directory specified in config.%s does not exist: %s' %
(v, getattr(config, v)))
def update_musicscores():
logger.debug("Updating all musicscores")
if not storage.exists(MANIFEST_PATH):
logger.debug("manifest.db not found, updating metadata")
meta_updater.update_database()
with db.CustomDB(meta_updater.get_manifestdb_path()) as manifest_conn:
all_musicscores = manifest_conn.execute_and_fetchall("""
SELECT name,hash FROM manifests WHERE (name LIKE "musicscores\_m___.bdb" ESCAPE '\\')
""")
all_musicscores = {_[0].split(".")[0]: _[1] for _ in all_musicscores}
if not _score_cache_db_exists():
_initialize_score_cache_db()
new_scores = all_musicscores.keys()
updated_scores = set()
else:
scores_meta = db.cachedb.execute_and_fetchall(
"SELECT score_id, score_hash FROM score_cache")
scores_meta = {_: __ for _, __ in scores_meta}
deleted_scores = set(scores_meta.keys()).difference(
all_musicscores.keys())
if len(deleted_scores) > 0:
logger.info(
"Found {} defunct musicscores, removing them...".format(
len(deleted_scores)))
for deleted_score in deleted_scores:
path = MUSICSCORES_PATH / "{}.db".format(deleted_score)
path.unlink()
new_scores = set(all_musicscores.keys()).difference(scores_meta.keys())
updated_scores = [
_ for _ in set(all_musicscores.keys()).intersection(
scores_meta.keys()) if scores_meta[_] != all_musicscores[_]
]
logger.info(
"Found {} musicscores, {} of them are new, {} are updated...".format(
len(all_musicscores), len(new_scores), len(updated_scores)))
if len(new_scores) + len(updated_scores) > 50:
logger.info("It will take some time to download, please wait...")
for musicscore_name in set(new_scores).union(set(updated_scores)):
musicscore_hash = all_musicscores[musicscore_name]
musicscore_response = cgss_query.get_db(musicscore_hash)
with storage.get_writer(
MUSICSCORES_PATH / "{}.db".format(musicscore_name),
'wb') as fwb:
fwb.write(decompress(musicscore_response.content))
db.cachedb.execute(
"""
INSERT OR REPLACE INTO score_cache (score_id, score_hash)
VALUES (?,?)
""", [musicscore_name, musicscore_hash])
db.cachedb.commit()
logger.info("All musicscores updated")
def init_level(name, level):
"""Initialize a given level of the search by saving all of the structures which need
to be evaluated."""
if not storage.exists(experiment_dir(name)):
raise RuntimeError('Experiment %s not yet initialized.' % name)
params = storage.load(params_file(name))
if level == 1:
init_structures = ['g']
else:
init_structures = storage.load(winning_structure_file(name, level - 1))
structure_pairs = list_structure_pairs(init_structures, params.rules, params.expand_noise)
storage.dump(structure_pairs, structures_file(name, level))
def init_level(name, level):
"""Initialize a given level of the search by saving all of the structures which need
to be evaluated."""
if not storage.exists(experiment_dir(name)):
raise RuntimeError('Experiment %s not yet initialized.' % name)
params = storage.load(params_file(name))
if level == 1:
init_structures = ['g']
else:
init_structures = storage.load(winning_structure_file(name, level - 1))
structure_pairs = list_structure_pairs(init_structures, params.rules,
params.expand_noise)
storage.dump(structure_pairs, structures_file(name, level))
def setup(update=False):
load_static()
if not (storage.exists(MANIFEST_PATH) and storage.exists(MASTERDB_PATH)):
update = True
from db import db
assert db
from network import meta_updater
if update:
meta_updater.update_database()
from logic.search import card_query
assert card_query
from network import music_updater
music_updater.update_musicscores()
from network import chart_cache_updater
chart_cache_updater.update_cache_scores()
from network import image_updater
assert image_updater
from logic.profile import profile_manager
assert profile_manager
from logic.search import indexer, search_engine
assert indexer
assert search_engine
from gui.viewmodels.chart_viewer import listener
assert listener
def update_image(card_id, sleep=0.1):
path = IMAGE_PATH / "{:06d}.png".format(card_id)
path32 = IMAGE_PATH32 / "{:06d}.jpg".format(card_id)
path64 = IMAGE_PATH64 / "{:06d}.jpg".format(card_id)
if not storage.exists(path):
time.sleep(sleep)
r = requests.get(FORMAT.format(card_id))
if r.status_code == 200:
with storage.get_writer(path, 'wb') as fwb:
for chunk in r:
fwb.write(chunk)
if not path64.exists() and not path32.exists():
img = Image.open(str(path)).convert('RGB')
if not path32.exists():
img.resize((32, 32), Image.ANTIALIAS).save(str(path32), format='JPEG')
if not path64.exists():
img.resize((64, 64), Image.ANTIALIAS).save(str(path64), format='JPEG')
def init_experiment(name, data_matrix, params, components=None, clean_data_matrix=None):
"""Initialize the structure search by saving the matrix, and possibly auxiliary
information, to files, and generating cross-validation splits."""
check_required_directories()
if not storage.exists(experiment_dir(name)):
storage.mkdir(experiment_dir(name))
storage.dump(params, params_file(name))
splits = nfold_cv(data_matrix.m, data_matrix.n, params.num_splits)
storage.dump(splits, splits_file(name))
if clean_data_matrix is not None:
storage.dump(clean_data_matrix, clean_data_file(name))
storage.dump(data_matrix, data_file(name))
if components is not None:
storage.dump(components, components_file(name))
def init_experiment(name, data_matrix, params, components=None, clean_data_matrix=None):
"""Initialize the structure search by saving the matrix, and possibly auxiliary
information, to files, and generating cross-validation splits."""
check_required_directories()
if not storage.exists(experiment_dir(name)):
storage.mkdir(experiment_dir(name))
storage.dump(params, params_file(name))
splits = nfold_cv(data_matrix.m, data_matrix.n, params.num_splits)
storage.dump(splits, splits_file(name))
if clean_data_matrix is not None:
storage.dump(clean_data_matrix, clean_data_file(name))
storage.dump(data_matrix, data_file(name))
if components is not None:
storage.dump(components, components_file(name))
def print_failures(name, outfile=sys.stdout):
params = storage.load(params_file(name))
failures = []
for level in range(1, params.search_depth + 1):
for _, structure in storage.load(structures_file(name, level)):
total_ok = 0
for split_id in range(params.num_splits):
for sample_id in range(params.num_samples):
fname = scores_file(name, level, structure, split_id, sample_id)
if storage.exists(fname):
row_loglik, col_loglik = storage.load(fname)
if np.all(np.isfinite(row_loglik)) and np.all(np.isfinite(col_loglik)):
total_ok += 1
if total_ok == 0:
failures.append(presentation.Failure(structure, level, True))
elif total_ok < params.num_splits * params.num_samples:
failures.append(presentation.Failure(structure, level, False))
presentation.print_failed_structures(failures, outfile)
def print_failures(name, outfile=sys.stdout):
params = storage.load(params_file(name))
failures = []
for level in range(1, params.search_depth + 1):
for _, structure in storage.load(structures_file(name, level)):
total_ok = 0
for split_id in range(params.num_splits):
for sample_id in range(params.num_samples):
fname = scores_file(name, level, structure, split_id, sample_id)
if storage.exists(fname):
row_loglik, col_loglik = storage.load(fname)
if np.all(np.isfinite(row_loglik)) and np.all(np.isfinite(col_loglik)):
total_ok += 1
if total_ok == 0:
failures.append(presentation.Failure(structure, level, True))
elif total_ok < params.num_splits * params.num_samples:
failures.append(presentation.Failure(structure, level, False))
presentation.print_failed_structures(failures, outfile)
def get_cachedb_path():
if not storage.exists(CACHEDB_PATH):
conn = sqlite3.connect(CACHEDB_PATH)
conn.close()
return CACHEDB_PATH
def get_masterdb_path():
if not storage.exists(MASTERDB_PATH):
logger.debug("master.db not found, triggering manifest updater")
_update_masterdb()
return MASTERDB_PATH
def get_manifestdb_path():
if not storage.exists(MANIFEST_PATH):
logger.debug("manifest.db not found, triggering manifest updater")
_update_manifest()
return MANIFEST_PATH
