def main(args):
print('Reading config...')
config = utilities.read_json_config(args.config, utilities.Task.inference)
print('Preparing output directory...')
output_dir = '{}/{}/inference'.format(config['dir']['output'],
config['name'])
utilities.create_dir(output_dir)
print('Preparing dataset...')
datasets = prepare_dataset(config, COMBINED_COL)
print('Preparing predictors...')
main_predictor = utilities.load(config['model']['main']['file'])
stringtable_predictor = utilities.load(
config['model']['stringtable']['file'])
prune_predictor = utilities.load(config['model']['prune']['file'])
otyrt_predictor = utilities.load(config['model']['otyrt']['file'])
print('Preparing other output dirs')
cdf_dir = '{}/cdf'.format(output_dir)
gnuplot_dir = '{}/gnuplot'.format(output_dir)
plot_dir = '{}/plot'.format(output_dir)
utilities.create_dir(cdf_dir)
utilities.create_dir(gnuplot_dir)
utilities.create_dir(plot_dir)
pbar = tqdm(range(len(datasets)))
for idx in pbar:
name = config['data'][idx]['name']
pbar.set_description(
'Outputting performance metrics for dataset {}'.format(name))
mse, r2 = test_predictor(datasets[idx], main_predictor,
stringtable_predictor, prune_predictor,
otyrt_predictor)
pbar.set_description('Generating diffs for dataset {}'.format(name))
diff = generate_diff(datasets[idx], main_predictor,
stringtable_predictor, prune_predictor,
otyrt_predictor)
pbar.set_description(
'Saving diffs for dataset {} prediction'.format(name))
sorted_indexes = save_diff(config, cdf_dir,
config['data'][idx]['name'], diff)
pbar.set_description(
'Creating plot for database {} prediction'.format(name))
save_plot(config['model'], config['data'][idx], cdf_dir, gnuplot_dir,
plot_dir, diff, sorted_indexes)
print('Saving combined plot')
save_plots(config, cdf_dir, gnuplot_dir, plot_dir)
def start_game_play(player, gameDisplay):
print(111111111111111111)
w, h = pygame.display.get_surface().get_size()
music_player = music.Music_Player()
music_player.play_ambtrack2()
gameDisplay.fill(config.black)
buttons = [
Button("BACK", config.white,
pygame.font.Font("assets/fonts/CHILLER.ttf", 70), (90, 60),
gameDisplay)
]
gui = pygame.Surface((w, h))
while True:
for event in pygame.event.get():
if event.type == pygame.KEYDOWN:
if event.key == pygame.K_ESCAPE:
global paused
paused = True
print(420420420)
utilities.pause()
#if s button is pressed save the character's stats
if event.key == pygame.K_s:
print('saved the game')
utilities.save(player)
#if l button is pressed load the character's stats
if event.key == pygame.K_l:
utilities.load()
player.pos = utilities.loaddata['pos']
player.health = utilities.loaddata['health']
player.actions = utilities.loaddata['actions']
player.items = utilities.loaddata['items']
player.hp = utilities.loaddata['hp']
elif (event.type == pygame.QUIT):
pygame.quit()
quit()
elif (event.type == pygame.MOUSEBUTTONDOWN and event.button == 1
and buttons[0].rect.collidepoint(pygame.mouse.get_pos())):
pass
# main_menu()
# for button in buttons:
# Button.check_Hover(button, gameDisplay)
pygame.display.update()
gui.fill(config.white)
gameDisplay.blit(gui, (0, 0))
game_start(player, gameDisplay)
clock.tick(15)
def load(folder, width, height):
print('Loading database from {}'.format(folder))
database_structure = utilities.get_database_structure(folder)
database = ImageDatabase(width, height, folder)
start_time = time.time()
chunks = database_structure.get_list_names()
total = len(chunks)
print('Loading {} chunks from {}'.format(
total, database_structure.image_folder))
database.images = []
for index, chunk in enumerate(chunks):
database.images += [
ImageItem(image, database.width, database.height)
for image in utilities.load(chunk)
]
utilities.print_progress(index + 1, total)
if settings.MAX_CHUNKS_USE and settings.MAX_CHUNKS_USE == index + 1:
print('Reached limit for max chunks use')
break
utilities.print_done(time.time() - start_time)
return database
def get_mc(update=False, verbose=False):
for year in year_tags:
fname = f'mc_{year}'
result = load(fname)
for dataset in datasets:
query = 'dataset dataset=/{}/{}*/NANOAODSIM'.format(
dataset, year_tags[year])
samples = get_das(query, verbose=verbose)
if not samples: continue
thesedatasets = set(s.split('/')[1] for s in samples)
for thisdataset in thesedatasets:
# NOTE: manually remove QCD_Pt-
if 'QCD_Pt-' in thisdataset: continue
if thisdataset not in result: result[thisdataset] = {}
sampleMap = result[thisdataset].get('files', {})
goodsamples = []
for sample in samples:
if not update and sample in sampleMap: continue
if 'Validation error' in sample: continue
if sample.split('/')[1] != thisdataset: continue
query = 'file dataset={}'.format(sample)
sampleMap[sample] = get_das(query, verbose=verbose)
goodsamples += [sample]
result[thisdataset] = {
'datasets': goodsamples,
'files': sampleMap
}
dump(fname, result)
def __init__(self, account):
"""
account is either a path to an account to load
or a dictionary with attribute "logins" as a list of login dictionaries
"""
from login import Login
from queue import Queue
argument = account
if type(account) is str:
argument = utilities.load(account + "/account.json")
self.savepath = account
for key in argument.keys():
setattr(self, key, argument[key])
self.queues = [] # this field is read-only, not loaded into memory
for i in range(len(self.logins)):
self.logins[i] = Login(self.logins[i])
try:
path = self.savepath + "/"
profs = self.logins[i].profiles()
self.queues.extend(
map(lambda x: Queue(x, path=path + Queue.profile_name(x)),
profs))
except AttributeError:
self.queues.extend(map(Queue, self.logins[i].profiles()))
self.subscriptions = argument["subscriptions"]
def predict(path, X):
a, b = load(path)
predicted_y = []
for x in X:
y = a + b * x
predicted_y.append(y)
return predicted_y
def predict(path, X):
a, b = load(path)
predicted_y = []
for x in X:
y = a + b * x
predicted_y.append(y)
predicted_y = np.array(predicted_y).reshape((len(predicted_y), 1))
return predicted_y
def load(path):
"""
Calls __init__ of dict["definition"]["type"] where dict is json at path
Returns Station subclass definition>type defined in stations.py
"""
import stations
diction = utilities.load(path)
sta = eval("stations." + diction["type"] + "(diction)")
sta.name = path.split("/")[-1].split(".")[0]
return sta
def __init__(self, profile, path=None):
self._prof = profile
part = path
if not path:
part = Queue.profile_name(profile)
try:
self.reserve = utilities.load(part)["reserve"]
except IOError:
self.reserve = []
def maybe_dump(path, fileset):
# if the file exists: read in the current data, and test for differences
replace = False
oldfileset = load(path)
for dataset in fileset:
oldfilelist = oldfileset.get(dataset, [])
filelist = fileset[dataset]
if Counter(oldfilelist) != Counter(filelist):
replace = True
if not replace: return
dump(path, fileset)
def update_existing_database(self):
print('Updating existing database')
meta_dict = utilities.load(self.database_meta_file)
files_to_add = [
file for file in self.files if file not in meta_dict['files']
]
total_files = len(files_to_add)
if total_files == 0:
print('There is no new images found ... [ done ]')
return
database_width = meta_dict['image width']
database_height = meta_dict['image height']
max_chunk_size = meta_dict['chunk size']
chunk_count = meta_dict['chunk count']
file_type = meta_dict['file type']
database_path = meta_dict['database path']
cache_images = []
num_of_cache_images = 0
data_paths = []
for index, file in enumerate(files_to_add):
cache_images.append(
DatabaseItem(file, database_width, database_height))
num_of_cache_images += 1
if num_of_cache_images >= max_chunk_size:
chunk_count += 1
data_path = database_path + os.path.sep + chunk_count + file_type
utilities.save(cache_images, data_path)
data_paths.append(data_path)
cache_images = []
num_of_cache_images = 0
utilities.print_progress(index + 1, total_files)
utilities.print_done()
# save remaining cached images
if cache_images: # if not empty
chunk_count += 1
data_path = database_path + os.path.sep + chunk_count + file_type
utilities.save(cache_images, data_path)
data_paths.append(data_path)
meta_dict['number of files'] += len(files_to_add)
meta_dict['files'] += files_to_add
meta_dict['paths'] += data_paths
meta_dict['chunk count'] += chunk_count
utilities.save(meta_dict, meta_dict['meta file path'])
print('--------------------------------')
# m4a转换成mp3
#mp3_audio = utilities.convert(audio)
# 生成wav文件名和路径
wav_audio = path + "/" + name + ".wav"
print "wav文件路径预设为： " + wav_audio
print('--------------------------------')
# 批量文件路径下的mp3转wav
#audioAnalysis.dirMp3toWavWrapper(path, 16000, 1)
# hmm分段，并生成segment文件
segFileName = path + "/" + name + ".segment"
print "seg文件路径预设为： " + segFileName
print('--------------------------------')
#[flagsInd, classesAll, acc, CM] = aS.hmmSegmentation(wav_audio, "data/hmmRadioSM", segFileName, True, '')
# 根据seg去除100秒以上的music
cmd = "mkdir " + path + "/" + name
sh.run(cmd)
# 记录分段起止时间
delimit_list = aS.segWAV(wav_audio)
# 读取转写结果
for i in (0, len(delimit_list) - 1):
records_list = utilities.load(path + "/" + name + "/" + name + "_" +
delimit_list[i][0] + "-" +
delimit_list[i][1] + ".json")
# 找到关键词出现的时间
feedback = utilities.feedback(records_list, keywords)
print "检查结果：" + str(feedback)
max_sig_idx = discovery_sig_df['ZA'].idxmax()
max_sig_data = discovery_sig_df.iloc[max_sig_idx]
max_sig = max_sig_data['ZA']
max_sig_cut = max_sig_data['t_cut']
max_sig_str = 'Max significance {:.2f} for cut value {:.4f}'.format(
max_sig, max_sig_cut)
print max_sig_str
return {'str': max_sig_str, 'cut': max_sig_cut, 'value': max_sig}
if __name__ == '__main__':
args = parse_args()
X_sig, X_bkg, w_sig, w_bkg = get_test_sig_bkg()
log.info('Loading classifier from {}'.format(args['clf']))
clf = utilities.load(args['clf'])
log.info('Loaded classifier\n{}'.format(clf))
log.info('Calculating predictions...')
p_sig = clf.decision_function(X_sig)
log.debug(p_sig)
log.debug('Mean: {:.5f}'.format(np.mean(p_sig)))
p_bkg = clf.decision_function(X_bkg)
log.debug(p_bkg)
log.debug('Mean: {:.5f}'.format(np.mean(p_bkg)))
ds = discovery_significance(p_sig,
p_bkg,
w_sig,
w_bkg,
t_range=(-2.5, 2.5))
# MATHEW LEWIS, JUNE 2020
import numpy as np
import tensorflow as tf
tf.keras.backend.set_floatx('float64')
from utilities import run,load
# load the test data
x_test = np.load('data/x_test.npy')
y_test = np.load('data/y_test.npy')
# run the model on some test data
result = run(load('models/model'),x_test[:10])
# compare the model's output with the correct answer
print('correct answer:\t\t',list(y_test[:10]))
print('models output:\t\t',result)
#!/usr/bin/env python
__author__ = 'cclamb'
import gzip
import utilities
import timeit
f = gzip.open('mnist-sd19.pkl.gz', 'rb')
data = utilities.load(f)
f.close()
utilities.create_hdf5_archive(data)
import numpy as np
import tensorflow as tf
tf.keras.backend.set_floatx('float64')
from utilities import run, load
# load the test data
x_test = np.load('data/x_test.npy')
y_test = np.load('data/y_test.npy')
# test the model
model = load('models/model')
score = model.evaluate(x_test, y_test, verbose=0)
print("%s: %.2f%%" % (model.metrics_names[1], score[1] * 100))
dr[dr < -self.box.hside] += self.box.side
norm_dr = np.linalg.norm(dr, axis=1)
norm_dr = norm_dr[norm_dr < self.interaction.cutoff]
u = self.interaction.potential(norm_dr,
self.interaction.parameters).sum()
return u
# start with one particle
r = np.array([[0, 0, 0]])
r = ut.load('last_pos.dat.xyz')
sim = Simulation(r, Box([L, L, L], [0, 0, 0]),
Interaction(twosteps, r3, parameters))
# insert particles
# r = moves.insert_hard_spheres(sim, N, r1, r3)
# ut.write_xyz("begin.xyz", sim.r)
move = moves.LocalMove(sim, delta, beta)
# x = np.arange(0,4,0.01)
# pl.plot(x, sim.interaction.potential(x,sim.interaction.parameters))
# pl.ylim(-2,2)
# pl.show()
for it in tqdm.tqdm(range(iterations)):
def _load(inputs):
return ImageItem(utilities.load(inputs[0]), inputs[1],
inputs[2]) # file width and height
def maybe_dump(path, fileset):
# if the file exists: read in the current data, and test for differences
replace = False
oldfileset = load(path)
for dataset in fileset:
oldfilelist = oldfileset.get(dataset, [])
filelist = fileset[dataset]
if Counter(oldfilelist) != Counter(filelist):
replace = True
if not replace: return
dump(path, fileset)
fulldatafileset = load('data')
fullmcfileset = load('mc')
years = ['2016', '2017', '2018']
for year in years:
outpath = 'filesets/{year}/{sample}'
fileset = {}
datafileset = {}
for s in fulldatafileset[year]:
thisfileset = {s: []}
for d in fulldatafileset[year][s]['datasets']:
thisfileset[s] += fulldatafileset[year][s]['files'][d]
maybe_dump(outpath.format(year=year, sample=s), thisfileset)
datafileset.update(thisfileset)
maybe_dump(outpath.format(year=year, sample='data'), datafileset)
def check_and_load_database(self):
# do some checking first
if not os.path.isdir(self.database_path):
raise ValueError('Database does not exists')
if not os.path.isfile(self.database_meta_file):
raise ValueError('Corrupted database: missing meta file')
meta_dict = utilities.load(self.database_meta_file)
database_chunk_size = meta_dict['chunk size']
if database_chunk_size != settings.DATABASE_CHUNK_SIZE:
raise ValueError(
'Existing database has different chunk size signature | database {} | program {}'
.format(database_chunk_size, settings.DATABASE_CHUNK_SIZE))
if settings.DATABASE_IMAGE_WIDTH != meta_dict['image width']:
raise ValueError(
'Existing database has different image width signature | database {} | program {}'
.format(settings.DATABASE_IMAGE_WIDTH,
meta_dict['image width']))
if settings.DATABASE_IMAGE_HEIGHT != meta_dict['image height']:
raise ValueError(
'Existing database has different image height signature | database {} | program {}'
.format(settings.DATABASE_IMAGE_HEIGHT,
meta_dict['image height']))
# now process the database
num_of_database_images = meta_dict['number of files']
num_of_files_found = len(self.files)
if num_of_database_images != num_of_files_found:
if num_of_database_images == 0:
self.create_and_load_database()
return
while True:
print('Files found: {} | Database size: {}'.format(
num_of_files_found, num_of_database_images))
print('[1] use existing database')
print('[2] create new database')
print('[3] update existing database')
res = input('Please enter a number:')
if res == '1':
break
elif res == '2':
self.create_and_load_database()
return
elif res == '3':
self.update_existing_database()
meta_dict = utilities.load(
self.database_meta_file) # update for below use
break
print('Please provide your answer as \'y\' or \'n\'')
images_list_paths = meta_dict['paths']
images_list_size = len(images_list_paths)
print('Loading database | {} | chunks {} | size per chunk {}'.format(
self.database_path, images_list_size, meta_dict['chunk size']))
self.loaded_image_items = []
for index, images_path in enumerate(images_list_paths):
try:
database_items = utilities.load(images_path)
except AttributeError as e:
raise ValueError(
'Database Object has different signature, did you change your code structure? | {}'
.format(e))
self.loaded_image_items += [
ImageItem(item, self.width, self.height)
for item in database_items
]
utilities.print_progress(index + 1, images_list_size)
utilities.print_done()
