def test_nnet_detector_returns_expected_results(path_to_nnet_config,
path_to_output_reference):
np.random.seed(0)
yass.set_config(path_to_nnet_config)
(standarized_path, standarized_params, channel_index,
whiten_filter) = preprocess.run()
scores, clear, collision = detect.run(standarized_path, standarized_params,
channel_index, whiten_filter)
path_to_scores = path.join(path_to_output_reference,
'detect_nnet_scores.npy')
path_to_clear = path.join(path_to_output_reference,
'detect_nnet_clear.npy')
path_to_collision = path.join(path_to_output_reference,
'detect_nnet_collision.npy')
ReferenceTesting.assert_array_almost_equal(scores,
path_to_scores,
decimal=4)
ReferenceTesting.assert_array_equal(clear, path_to_clear)
ReferenceTesting.assert_array_equal(collision, path_to_collision)
clean_tmp()
def handle_notification(r, user):
if not user.can_do("notify", r.builder):
log.alert("user %s is not allowed to notify:%s" %
(user.login, r.builder))
q = B_Queue(path.req_queue_file)
q.lock(0)
q.read()
not_fin = filter(lambda (r): not r.is_done(), q.requests)
r.apply_to(q)
for r in not_fin:
if r.is_done():
util.clean_tmp(path.srpms_dir + '/' + r.id)
now = time.time()
def leave_it(r):
# for ,,done'' set timeout to 4d
if r.is_done() and r.time + 4 * 24 * 60 * 60 < now:
return False
# and for not ,,done'' set it to 20d
if r.time + 20 * 24 * 60 * 60 < now:
util.clean_tmp(path.srpms_dir + '/' + r.id)
return False
return True
q.requests = filter(leave_it, q.requests)
q.write()
q.dump(path.queue_stats_file)
q.dump_html(path.queue_html_stats_file)
q.write_signed(path.req_queue_signed_file)
q.unlock()
def test_can_detect_with_nnet(path_to_nnet_config):
yass.set_config(path_to_nnet_config)
standarized_path, standarized_params, whiten_filter = preprocess.run()
scores, clear, collision = detect.run(standarized_path, standarized_params,
whiten_filter)
clean_tmp()
def test_example_works_pip_and_dict(path_to_config_sample):
with open(path_to_config_sample) as f:
cfg = yaml.load(f)
pipeline.run(cfg)
clean_tmp()
def test_templates_returns_expected_results(path_to_threshold_config,
path_to_data_folder):
np.random.seed(0)
yass.set_config(path_to_threshold_config)
(standarized_path, standarized_params, channel_index,
whiten_filter) = preprocess.run()
(score, spike_index_clear,
spike_index_all) = detect.run(standarized_path,
standarized_params,
channel_index,
whiten_filter)
spike_train_clear, tmp_loc, vbParam = cluster.run(score, spike_index_clear)
(templates_, spike_train,
groups, idx_good_templates) = templates.run(spike_train_clear, tmp_loc)
path_to_templates = path.join(path_to_data_folder,
'output_reference',
'templates.npy')
ReferenceTesting.assert_array_equal(templates_, path_to_templates)
clean_tmp()
def handle_notification(r, user):
if not user.can_do("notify", r.builder):
log.alert("user %s is not allowed to notify:%s" % (user.login, r.builder))
q = B_Queue(path.req_queue_file)
q.lock(0)
q.read()
not_fin = filter(lambda (r): not r.is_done(), q.requests)
r.apply_to(q)
for r in not_fin:
if r.is_done():
util.clean_tmp(path.srpms_dir + '/' + r.id)
now = time.time()
def leave_it(r):
# for ,,done'' set timeout to 4d
if r.is_done() and r.time + 4 * 24 * 60 * 60 < now:
return False
# and for not ,,done'' set it to 20d
if r.time + 20 * 24 * 60 * 60 < now:
util.clean_tmp(path.srpms_dir + '/' + r.id)
return False
return True
q.requests = filter(leave_it, q.requests)
q.write()
q.dump(path.queue_stats_file)
q.dump_html(path.queue_html_stats_file)
q.write_signed(path.req_queue_signed_file)
q.unlock()
def test_can_detect_with_threshold(path_to_threshold_config):
yass.set_config(path_to_threshold_config)
(standarized_path, standarized_params, channel_index,
whiten_filter) = preprocess.run()
scores, clear, collision = detect.run(standarized_path, standarized_params,
channel_index, whiten_filter)
clean_tmp()
def clean_dir(path, max):
curtime = time.time()
for i in os.listdir(path):
if curtime - os.path.getmtime(path + '/' + i) > max:
if os.path.isdir(path + '/' + i):
util.clean_tmp(path + '/' + i)
else:
os.unlink(path + '/' + i)
def clean_dir(path, max):
curtime=time.time()
for i in os.listdir(path):
if curtime - os.path.getmtime(path+'/'+i) > max:
if os.path.isdir(path+'/'+i):
util.clean_tmp(path+'/'+i)
else:
os.unlink(path+'/'+i)
def test_decovnolution(path_to_config):
yass.set_config('tests/config_nnet.yaml')
clear_scores, spike_index_clear, spike_index_collision = preprocess.run()
(spike_train_clear, templates,
spike_index_collision) = process.run(clear_scores, spike_index_clear,
spike_index_collision)
deconvolute.run(spike_train_clear, templates, spike_index_collision)
clean_tmp()
def leave_it(r):
# for ,,done'' set timeout to 4d
if r.is_done() and r.time + 4 * 24 * 60 * 60 < now:
return False
# and for not ,,done'' set it to 20d
if r.time + 20 * 24 * 60 * 60 < now:
util.clean_tmp(path.srpms_dir + '/' + r.id)
return False
return True
def leave_it(r):
# for ,,done'' set timeout to 4d
if r.is_done() and r.time + 4 * 24 * 60 * 60 < now:
return False
# and for not ,,done'' set it to 20d
if r.time + 20 * 24 * 60 * 60 < now:
util.clean_tmp(path.srpms_dir + '/' + r.id)
return False
return True
def test_threshold_pipeline_returns_expected_results(path_to_threshold_config,
path_to_data_folder):
spike_train = pipeline.run(path_to_threshold_config, clean=True)
path_to_reference = path.join(path_to_data_folder,
'output_reference',
'threshold_spike_train.npy')
ReferenceTesting.assert_array_equal(spike_train, path_to_reference)
clean_tmp()
def test_cluster(path_to_threshold_config):
yass.set_config(path_to_threshold_config)
(standarized_path, standarized_params, whiten_filter) = preprocess.run()
(score, spike_index_clear,
spike_index_all) = detect.run(standarized_path, standarized_params,
whiten_filter)
cluster.run(score, spike_index_clear)
clean_tmp()
def test_decovnolution(path_to_threshold_config):
yass.set_config('tests/config_nnet.yaml')
(standarized_path, standarized_params, whiten_filter) = preprocess.run()
(score, spike_index_clear,
spike_index_all) = detect.run(standarized_path, standarized_params,
whiten_filter)
spike_train_clear, tmp_loc, vbParam = cluster.run(score, spike_index_clear)
(templates_, spike_train, groups,
idx_good_templates) = templates.run(spike_train_clear, tmp_loc)
deconvolute.run(spike_index_all, templates_)
clean_tmp()
def test_templates(path_to_threshold_config):
yass.set_config(path_to_threshold_config)
(standarized_path, standarized_params, channel_index,
whiten_filter) = preprocess.run()
(score, spike_index_clear,
spike_index_all) = detect.run(standarized_path,
standarized_params,
channel_index,
whiten_filter)
spike_train_clear, tmp_loc, vbParam = cluster.run(
score, spike_index_clear)
templates.run(spike_train_clear, tmp_loc)
clean_tmp()
def test_preprocess_returns_expected_results(path_to_threshold_config,
path_to_output_reference):
yass.set_config(path_to_threshold_config)
standarized_path, standarized_params, whiten_filter = preprocess.run()
# load standarized data
standarized = np.fromfile(standarized_path,
dtype=standarized_params['dtype'])
path_to_standarized = path.join(path_to_output_reference,
'preprocess_standarized.npy')
path_to_whiten_filter = path.join(path_to_output_reference,
'preprocess_whiten_filter.npy')
ReferenceTesting.assert_array_almost_equal(standarized,
path_to_standarized)
ReferenceTesting.assert_array_almost_equal(whiten_filter,
path_to_whiten_filter)
clean_tmp()
def test_cluster_returns_expected_results(path_to_threshold_config,
path_to_data_folder):
np.random.seed(0)
yass.set_config(path_to_threshold_config)
(standarized_path, standarized_params, whiten_filter) = preprocess.run()
(score, spike_index_clear,
spike_index_all) = detect.run(standarized_path, standarized_params,
whiten_filter)
spike_train, tmp_loc, vbParam = cluster.run(score, spike_index_clear)
path_to_spike_train = path.join(path_to_data_folder, 'output_reference',
'cluster_spike_train.npy')
path_to_tmp_loc = path.join(path_to_data_folder, 'output_reference',
'cluster_tmp_loc.npy')
ReferenceTesting.assert_array_equal(spike_train, path_to_spike_train)
ReferenceTesting.assert_array_equal(tmp_loc, path_to_tmp_loc)
clean_tmp()
def teardown_function(function):
reset_config()
clean_tmp()
def test_example_works_default_pipeline(path_to_config_sample):
pipeline.run(path_to_config_sample)
clean_tmp()
def teardown_function(function):
clean_tmp()
def test_example_works(path_to_config_sample, path_to_output):
cli._run_pipeline(path_to_config_sample, path_to_output)
clean_tmp()
def test_can_preprocess_with_nnet(path_to_nn_config):
yass.set_config(path_to_nn_config)
clear_scores, spike_index_clear, spike_index_collision = preprocess.run()
clean_tmp()
def tmp_folder():
make_tmp()
yield os.path.join(PATH_TO_TESTS, 'data/tmp/')
clean_tmp()
