def process_and_save_files(self):
"""
process files in self.files, according to its width and height
:return: None
"""
print('Processing and saving files ...')
start_time = time.time()
self.structure.remove_existing_files()
self.structure.make_folders()
self.images = []
total = len(self.files)
files_chunks = []
start = 0
while start < total - 1:
end = start + settings.MAX_CACHE_PROCESSED_IMAGES
files_chunks.append(self.files[start:end])
start = end
total_chunks = len(files_chunks)
for chunk_index, chunk in enumerate(files_chunks):
processed_images = []
total_files = len(chunk)
for file_index, file in enumerate(chunk):
database_item = DatabaseImageItem(file)
processed_images.append(database_item)
self.images.append(
ImageItem(database_item, self.width, self.height))
utilities.print_progress(file_index + 1,
total_files,
curr_chunk=chunk_index + 1,
total_chunks=total_chunks)
utilities.save(processed_images,
self.structure.get_list_name(chunk_index))
utilities.print_done(time.time() - start_time)
def _make_sticker(self,
label,
title="",
pixel_threshold=0.01,
save_threshold=0.9):
basic_im = np.copy(self.start)
for i in range(self.num_rows):
for j in range(self.num_rows):
for k in range(3):
if self.probarray[i, j, k, label] > pixel_threshold:
basic_im[self.fringe + i * self.pixelsize:self.fringe +
(i + 1) * self.pixelsize,
self.fringe + j * self.pixelsize:self.fringe +
(j + 1) * self.pixelsize, k] = 255
if np.max(basic_im - self.start) > 0:
prob = np.array(save_and_query(basic_im, "temp.png"))
self.queries += 1
sleep(1)
print("Confidence for Sticker on label " + str(label) + ": " +
str(prob[label]))
else:
prob = np.zeros(LABEL_AMOUNT)
if prob[label] > save_threshold:
basic_im = unstick(basic_im, self.start)
save_url = STICKER_DIRECTORY + "/" + str(label) + "/" + title +\
str(pixel_threshold) + str(prob[label]) + ".png"
save(basic_im, save_url)
print("Sticker saved under " + save_url)
try:
remove("temp.png")
except FileNotFoundError:
pass
return basic_im
def create_class():
global school_data
class_room = input("Введите класс или Enter для возврата в предыдущее меню\n:")
if class_room:
school_data['classes'].append(class_room)
save(school_data, 'school.json')
input('Класс успешно создан, нажмите Enter для возврата в предыдущее меню\n:')
else:
return
def create_database(self):
print('Creating database | {} | '.format(self.database_path), end='')
# remove existing database if any
if os.path.isdir(self.database_path):
shutil.rmtree(self.database_path)
os.mkdir(self.database_path)
pool = Pool()
file_size = len(self.files)
database_item_generator = functools.partial(
DatabaseItem,
width=settings.DATABASE_IMAGE_WIDTH,
height=settings.DATABASE_IMAGE_HEIGHT)
print('total {} '.format(file_size))
data_paths = []
cache_images = []
cache_images_size = 0
print(' >>> waiting for arrival of chunks ...', end='')
for index, items in enumerate(
pool.imap_unordered(database_item_generator, self.files)):
cache_images.append(items) # database item
cache_images_size += 1
if (cache_images_size >=
settings.DATABASE_CHUNK_SIZE) or index + 1 == file_size:
data_path = self.get_save_image_path()
data_paths.append(data_path)
utilities.save(cache_images, data_path)
cache_images = []
cache_images_size = 0
utilities.print_progress(index + 1, file_size)
database_meta = {
'number of files': file_size,
'chunk size': settings.DATABASE_CHUNK_SIZE,
'files': self.files,
'paths': data_paths,
'image width': settings.DATABASE_IMAGE_WIDTH,
'image height': settings.DATABASE_IMAGE_HEIGHT,
'chunk count': self.save_image_count,
'file type': settings.DATABASE_FILE_TYPE,
'database path': self.database_path,
'meta file path': self.database_meta_file
}
# save meta
utilities.save(database_meta, self.database_meta_file)
utilities.print_done()
def create_class():
global school_data
class_room = input(
"Введите класс или Enter для возврата в предыдущее меню\n:")
if class_room:
school_data['classes'].append(class_room)
save(school_data, 'school.json')
input(
'Класс успешно создан, нажмите Enter для возврата в предыдущее меню\n:'
)
else:
return
def start_game_play(player, gameDisplay):
print(111111111111111111)
w, h = pygame.display.get_surface().get_size()
music_player = music.Music_Player()
music_player.play_ambtrack2()
gameDisplay.fill(config.black)
buttons = [
Button("BACK", config.white,
pygame.font.Font("assets/fonts/CHILLER.ttf", 70), (90, 60),
gameDisplay)
]
gui = pygame.Surface((w, h))
while True:
for event in pygame.event.get():
if event.type == pygame.KEYDOWN:
if event.key == pygame.K_ESCAPE:
global paused
paused = True
print(420420420)
utilities.pause()
#if s button is pressed save the character's stats
if event.key == pygame.K_s:
print('saved the game')
utilities.save(player)
#if l button is pressed load the character's stats
if event.key == pygame.K_l:
utilities.load()
player.pos = utilities.loaddata['pos']
player.health = utilities.loaddata['health']
player.actions = utilities.loaddata['actions']
player.items = utilities.loaddata['items']
player.hp = utilities.loaddata['hp']
elif (event.type == pygame.QUIT):
pygame.quit()
quit()
elif (event.type == pygame.MOUSEBUTTONDOWN and event.button == 1
and buttons[0].rect.collidepoint(pygame.mouse.get_pos())):
pass
# main_menu()
# for button in buttons:
# Button.check_Hover(button, gameDisplay)
pygame.display.update()
gui.fill(config.white)
gameDisplay.blit(gui, (0, 0))
game_start(player, gameDisplay)
clock.tick(15)
def main(args):
print('Reading config...')
config = utilities.read_json_config(args.config, utilities.Task.train)
train_type = str(args.type)
print('Preparing output directory...')
output_dir = '{}/{}/train/{}'.format(config['dir']['output'], config['name'], train_type)
utilities.create_dir(output_dir)
print('Preparing dataset...')
dataset = prepare_dataset(config, train_type, get_data_col(args.type))
print('Preparing trainers...')
trainers = prepare_trainer(config)
print('There are {} models that needs to be trained'.format(len(trainers)))
print()
print('Training predictors...')
predictors = train_predictor(config, trainers, dataset)
print(predictors)
print()
print('Test predictors...')
tests = test_predictor(predictors, dataset)
print('Preparing other output dirs')
cdf_dir = '{}/cdf'.format(output_dir)
gnuplot_dir = '{}/gnuplot'.format(output_dir)
plot_dir = '{}/plot'.format(output_dir)
model_dir = '{}/model'.format(output_dir)
utilities.create_dir(cdf_dir)
utilities.create_dir(gnuplot_dir)
utilities.create_dir(plot_dir)
utilities.create_dir(model_dir)
print('Generate diff and plots...')
pbar = tqdm(predictors)
for predictor in pbar:
pbar.set_description('Generate diffs for {}'.format(predictor))
diff = generate_diff(config, predictors, predictor, dataset)
pbar.set_description('Saving diffs for {}'.format(predictor))
sorted_indexes = save_diff(config, cdf_dir, predictor, diff)
pbar.set_description('Creating plot for {}'.format(predictor))
save_plot(config, cdf_dir, gnuplot_dir, plot_dir, predictor, diff, sorted_indexes)
pbar.set_description('Saving model for {}'.format(predictor))
utilities.save('{}/{}.joblib'.format(model_dir, predictor), predictors[predictor])
def infer_save_frames(input_frames_path, out_frames_path, every_nth=1):
frame_files = sorted(glob.glob(input_frames_path + '/*'))
num_frames = len(frame_files)
detect_time = 0
recognize_time = 0
print('Detecting and recognizing text from {} frames: {}'.format(
num_frames, str(datetime.now())))
wordBB = None
score = None
text = None
for index, filename in tqdm(enumerate(frame_files), total=num_frames):
out_name = out_frames_path + '/out_{0:04d}.png'.format(index)
if (index % every_nth == 0):
wordBB, score = detection.detect(filename)
if score.shape[0] == 0:
wordBB = None
score = None
else:
text = recognition.recognize(filename, wordBB)
utilities.save(filename, wordBB, text, out_name)
def update_existing_database(self):
print('Updating existing database')
meta_dict = utilities.load(self.database_meta_file)
files_to_add = [
file for file in self.files if file not in meta_dict['files']
]
total_files = len(files_to_add)
if total_files == 0:
print('There is no new images found ... [ done ]')
return
database_width = meta_dict['image width']
database_height = meta_dict['image height']
max_chunk_size = meta_dict['chunk size']
chunk_count = meta_dict['chunk count']
file_type = meta_dict['file type']
database_path = meta_dict['database path']
cache_images = []
num_of_cache_images = 0
data_paths = []
for index, file in enumerate(files_to_add):
cache_images.append(
DatabaseItem(file, database_width, database_height))
num_of_cache_images += 1
if num_of_cache_images >= max_chunk_size:
chunk_count += 1
data_path = database_path + os.path.sep + chunk_count + file_type
utilities.save(cache_images, data_path)
data_paths.append(data_path)
cache_images = []
num_of_cache_images = 0
utilities.print_progress(index + 1, total_files)
utilities.print_done()
# save remaining cached images
if cache_images: # if not empty
chunk_count += 1
data_path = database_path + os.path.sep + chunk_count + file_type
utilities.save(cache_images, data_path)
data_paths.append(data_path)
meta_dict['number of files'] += len(files_to_add)
meta_dict['files'] += files_to_add
meta_dict['paths'] += data_paths
meta_dict['chunk count'] += chunk_count
utilities.save(meta_dict, meta_dict['meta file path'])
def save_all():
save(students_data, 'Students.json')
save(teachers_data, 'Teachers.json')
save(school_data, 'school.json')
min_weight_fraction_leaf=min_weight_fraction_leaf,
max_depth=max_depth,
max_features=max_features,
verbose=2)
if __name__ == '__main__':
TRAIN_DATA = 'atlas-higgs-challenge-2014-v2_train.csv'
args = parse_args()
save_arg = args.pop('save')
log.info('Reading data from {}'.format(TRAIN_DATA))
X, y, w = utilities.get_data_labels_weights(TRAIN_DATA)
log.info('Renormalising weights')
w_norm = utilities.normalise_weights(w, y)
gbc = make_GBC(**args)
log.info('Created classifier\n{}'.format(gbc))
log.info('Training classifier...')
start_time = time.time()
mean_score = gbc.fit(X, y, sample_weight=w_norm)
end_time = time.time()
log.info('Completed in {}'.format(
utilities.delta_time(start_time, end_time)))
log.info('Saving classifier as {}'.format(save_arg))
utilities.save(gbc, save_arg)
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import mpi_resistor_model as resistor_model
from functools import partial
import numpy as np
import networkx as nx
steps = 40000
selfishness = lambda: np.random.rand()
gain = lambda: 0.0005
con = resistor_model.Controller(100, Ri=2, R=200, U=10, comm_error=0.01,
overcurrent=4.5,min_gain=gain,turns=steps,generate_graph=True,
,selfishness=selfishness)
con.is_constrained=False
con.run()
con.collectAgents()
if con.rank == 0:
print "computation finished."
import utilities
data = {'steps':steps,'Ri':con.Ri,'R':con.R/con._N,'U':con.U,
'comm_error':con.comm_error,'overcurrent':con.overcurrent,
'min_gain':0.0005,'constrained':con.is_constrained,
'N_agents':con.N_agents,'horizon':con.trend_horizon}
utilities.save(con,data,'test.npz')
def save(image, structure):
filename = structure.get_image_filename(image.filename)
utilities.save(image, filename)
def save_all():
save(students_data, 'Students.json')
save(teachers_data, 'Teachers.json')
save(school_data, 'school.json')
# MATHEW LEWIS, JUNE 2020
import numpy as np
import tensorflow as tf
tf.keras.backend.set_floatx('float64')
from utilities import save, compAndTrain
# get the data
x_train = np.load('data/x_train.npy')
y_train = np.load('data/y_train.npy')
# build the model
# this initializes the model
model = tf.keras.models.Sequential()
# this adds a layer with 128 nodes, dense means they are all connected
model.add(tf.keras.layers.Dense(128, activation='relu'))
# dropout means when training we drop out some of the edges in the graph
# this makes the model more robust
model.add(tf.keras.layers.Dropout(0.2))
# another 10 nodes (10 because the demo classifies then numbers 1 through 10)
model.add(tf.keras.layers.Dense(10))
# compile and train the model
compAndTrain(model, x_train, y_train, 4)
# save the model
save(model, 'models/model')
def character_selection(gameDisplay):
print("selecting characters")
w, h = pygame.display.get_surface().get_size()
music_player = music.Music_Player()
music_player.play_ambtrack9()
gameDisplay.fill(config.black)
buttons = [Button("BACK", config.white, pygame.font.Font("assets/fonts/CHILLER.ttf", 70), (90, 60), gameDisplay)]
character_list = create_all_characters()
selection_gui = pygame.Surface((w, h))
image_list, character_types = get_image_list()
image_rect_list = []
num_of_images = len(image_list) + 1
start_x = 0
start_y = 0
x_offset = int(w / num_of_images)
char_detail_surf, char_detail_rect = pygame.Surface((0,0)), pygame.Surface((0,0)) # init to random surface to avoid crash on line 2016
for elem in image_list:
transformed_image = transform_image(elem, w, h, num_of_images)
image_rect_list.append(set_image_location(transformed_image, start_x, start_y))
start_x += x_offset
char_detail_offset = start_x
while True:
for event in pygame.event.get():
if (event.type == pygame.QUIT):
pygame.quit()
quit()
elif (selection_gui.get_rect().collidepoint(pygame.mouse.get_pos())):
index = 0
for char_image, char_rect in image_rect_list:
if (char_rect.collidepoint(pygame.mouse.get_pos())):
character = Character(character_types[index])
size = (x_offset, h)
char_detail_surf = get_player_stats(character, size)
break
index += 1
#back button when clicked returns to main menu
if (event.type == pygame.MOUSEBUTTONDOWN and event.button == 1 and buttons[0].rect.collidepoint(
pygame.mouse.get_pos())):
return
elif (event.type == pygame.MOUSEBUTTONDOWN and event.button == 1):
index = 0
for char_image, char_rect in image_rect_list:
if (char_rect.collidepoint(event.pos)):
print("character selected")
character = Character(character_types[index])
player = Player(character)
save(player)
game_start(player, gameDisplay)
index += 1
elif (selection_gui.get_rect().collidepoint(pygame.mouse.get_pos())):
index = 0
for char_image, char_rect in image_rect_list:
if (char_rect.collidepoint(pygame.mouse.get_pos())):
character = Character(character_types[index])
size = (x_offset, h)
char_detail_surf = get_player_stats(character, size)
break
index += 1
for button in buttons:
Button.check_Hover(button, gameDisplay)
pygame.display.update()
selection_gui.fill(config.white)
selection_gui.blit(char_detail_surf, (char_detail_offset, 10))
for image, image_rect in image_rect_list:
selection_gui.blit(image, image_rect)
gameDisplay.blit(selection_gui, (0, 0))
TextSurf, TextRect = render_text("Choose a Character", config.SPOOKY_BIG_FONT, config.red)
TextRect.center = ((round(config.display_width / 2)), (round(config.display_height * .9)))
gameDisplay.blit(TextSurf, TextRect)
clock.tick(15)
layerStack,
memSize=config['agent']['replayMemorySize'],
stackedStateLength=config['agent']['stackedStateLength'],
stateScaleFactor=config['agent']['stateScaleFactor'],
epsilonPolicy=epsilonPolicy,
optimizer=optimizer,
loss=config['model']['lossFunction'],
batchSize=config['model']['batchSize'],
modelName=config['model']['name'])
# If required load the old model for futher learning
if config['paths']['initialModelName'] != False:
modelToLoad = os.path.join(config['paths']['savesDir'],
env.unwrapped.spec.id, 'models',
config['paths']['initialModelName'])
agent.loadModel(modelToLoad)
# Load replay memory if needed
if config['paths']['initialReplayMemoryName'] != False:
replaysToLoad = os.path.join(config['paths']['savesDir'],
env.unwrapped.spec.id, 'replays',
config['paths']['initialReplayMemoryName'])
agent.loadMemory(replaysToLoad)
#===================================================================================#
#====================================== Training ===================================#
#===================================================================================#
if config['mode'] == 'training':
session(env, config, agent, lambda name: save(name, savesDir, agent))
elif config['mode'] == 'evaluation':
evaluation(env, config, agent)
def train(args):
start_epoch = 0
corpus_file = args.datadir+'/train.'
util.trace('start training...')
chainer.global_config.train = True
chainer.global_config.use_cudnn = 'always'
chainer.global_config.type_check = True
util.trace('chainer config: {}'.format(chainer.global_config.__dict__))
util.trace('load vocab...')
vocab_file = args.datadir+'/vocabulary.'
source_vocab = util.Vocabulary.load(vocab_file+args.sourcelang)
target_vocab = util.Vocabulary.load(vocab_file+args.targetlang)
"""
util.trace('make vocab...')
source_vocab = util.Vocabulary.make(corpus_file+args.sourcelang, 3000)
target_vocab = util.Vocabulary.make(corpus_file+args.targetlang, 3000)
"""
if args.gensim_mode == 'make':
util.trace('making word2vec...')
src_word2vec = util.make_word2vec(corpus_file+args.sourcelang, args.edim)
tgt_word2vec = util.make_word2vec(corpus_file+args.targetlang, args.edim)
util.save(src_word2vec, args.datadir+'/src_word2vec.'+args.sourcelang)
util.save(tgt_word2vec, args.datadir+'/tgt_word2vec.'+args.targetlang)
elif args.gensim_mode == 'load':
util.trace('loading word2vec...')
src_word2vec = util.load_word2vec(args.datadir+'/src_word2vec.'+args.sourcelang)
tgt_word2vec = util.load_word2vec(args.datadir+'/tgt_word2vec.'+args.targetlang)
elif args.gensim_mode == 'not':
util.trace('do not use word2vec')
src_word2vec = None
tgt_word2vec = None
util.trace('making model...')
#initialize model
NMTmodel = nmt_model.BahdanauNMT(source_vocab, target_vocab, args, src_word2vec, tgt_word2vec)
if args.gpunum >= 0:
import cupy as xp
chainer.cuda.check_cuda_available()
chainer.cuda.get_device(args.gpunum).use()
NMTmodel.to_gpu()
util.trace('use GPU id: {}'.format(args.gpunum))
else:
import numpy as xp
args.gpunum = -1
util.trace('without GPU')
util.trace('random seed: {}'.format(args.seed_num))
np.random.seed(args.seed_num)
xp.random.seed(args.seed_num)
random.seed(args.seed_num)
optim = args.optim
#this is change
optim = chainer.optimizers.AdaGrad(lr=args.lr)
optim.setup(NMTmodel)
optim.add_hook(chainer.optimizer.GradientClipping(args.grad_clip))
for epoch in range(start_epoch, args.epoch):
util.trace('Epoch {}/{}'.format(epoch+1, args.epoch))
accum_loss = 0.0
num_sent = 0
for batch_src, batch_tgt in util.miniBatch(corpus_file+args.sourcelang, corpus_file+args.targetlang,\
source_vocab, target_vocab, args.batch, args.pooling):
NMTmodel.zerograds()
loss, batch_hyp = NMTmodel(batch_src, batch_tgt)
accum_loss += loss.data
loss.backward()
optim.update()
for src, tgt, hyp in zip(util.convert_b2w(batch_src, source_vocab), util.convert_b2w(batch_tgt, target_vocab), \
util.convert_b2w(batch_hyp, target_vocab)):
util.trace('Epoch {}/{}, {} sent'.format(epoch+1, args.epoch, num_sent+1))
util.trace('src: {}'.format(src))
util.trace('tgt: {}'.format(tgt))
util.trace('hyp: {}'.format(hyp))
num_sent += 1
util.trace('accum_loss: {}'.format(accum_loss))
util.trace('Save model ...')
model_name = '{}.{:03d}'.format(args.name, epoch+1)
chainer.serializers.save_npz(args.savedir+'/{}.weights'.format(model_name), NMTmodel)
chainer.serializers.save_npz(args.savedir+'/{}.optimizer'.format(model_name), optim)