def construct_model(self):
self.q_net = self.build_net()
self.model = Model(net=self.q_net,
loss=MSE(),
optimizer=RMSProp(self.args.lr))
# Target network
self.target_q_net = self.build_net()
def read(self, pb_path: str) -> Model:
"""Read TF file and load model.
Parameters
----------
pb_path : str
Path to TF file
Returns
-------
model : Model
Loaded model
"""
model = Model()
# load tensorflow model
graph_def = graph_pb2.GraphDef()
try:
f = open(path.abspath(pb_path), "rb")
graph_def.ParseFromString(f.read())
f.close()
except IOError:
print("Could not open file. Creating a new one.")
# import graph
model.graph = Importer.make_graph(graph_def)
return model
def main(_):
pp.pprint(FLAGS.__flags)
if FLAGS.threads < np.inf:
sess_config = tf.ConfigProto(
intra_op_parallelism_threads=FLAGS.threads)
sess_config.gpu_options.per_process_gpu_memory_fraction = FLAGS.gpu_mem
else:
sess_config = tf.ConfigProto()
if 'mmd' in FLAGS.model:
from core.model import MMD_GAN as Model
elif FLAGS.model == 'wgan_gp':
from core.wgan_gp import WGAN_GP as Model
elif 'cramer' in FLAGS.model:
from core.cramer import Cramer_GAN as Model
with tf.Session(config=sess_config) as sess:
if FLAGS.dataset == 'mnist':
gan = Model(sess,
config=FLAGS,
batch_size=FLAGS.batch_size,
output_size=28,
c_dim=1,
data_dir=FLAGS.data_dir)
elif FLAGS.dataset == 'cifar10':
gan = Model(sess,
config=FLAGS,
batch_size=FLAGS.batch_size,
output_size=32,
c_dim=3,
data_dir=FLAGS.data_dir)
elif FLAGS.dataset in ['celebA', 'lsun']:
gan = Model(sess,
config=FLAGS,
batch_size=FLAGS.batch_size,
output_size=FLAGS.output_size,
c_dim=3,
data_dir=FLAGS.data_dir)
else:
gan = Model(sess,
batch_size=FLAGS.batch_size,
output_size=FLAGS.output_size,
c_dim=FLAGS.c_dim,
data_dir=FLAGS.data_dir)
if FLAGS.is_train:
gan.train()
elif FLAGS.print_pca:
gan.print_pca()
elif FLAGS.visualize:
gan.load_checkpoint()
visualize(sess, gan, FLAGS, 2)
else:
gan.get_samples(FLAGS.no_of_samples, layers=[-1])
if FLAGS.log:
sys.stdout = gan.old_stdout
gan.log_file.close()
gan.sess.close()
def test_import_file_updates_db_with_players():
"""
End to end test asserting that a HTML file is parsed and imported into an
in-memory SQLite database.
"""
model = Model(echo=False)
# open the HTML file
file = open(
path.join(PROJECT_SOURCE_PATH, 'test', 'resources', 'squad.html'), 'r'
)
html = file.read()
file.close()
# parse the players
players = parse_players(html)
with model.session_scope() as session:
# open a session and import our HTML file
html_file = HTMLFile.from_file(
path.join(PROJECT_SOURCE_PATH, 'test', 'resources', 'squad.html'))
model.import_file(html_file)
# retrieve the team from the database
# and check that it has imported correctly
new_team = session.query(Team).first()
assert_equal(len(new_team.players), len(players))
def test():
# 开始计时
start = time.clock()
test_date = time.time()
data_path = './data/processed/prt_mfd_7.csv'
save_path = './test/purchase/'
col_start = 1
col_end = 4
train_begin = 0
train_end = 397
time_step = 15
rnn_unit = 10
lr = 0.0006
run_times = 100
# ---------------------------------------------
if not os.path.exists(save_path):
os.makedirs(save_path)
# ---------------------------------------------
model = Model(save_path, test_date, data_path, col_start, col_end,
train_begin, train_end, time_step, rnn_unit, lr, run_times)
test_y, test_predict = model.get_test_result()
# -------------------------------
end = time.clock()
spend = end - start
utils = Utils()
test_score = utils.get_score(test_y, test_predict)
describe = 'purchase(1308_1408,[purchase,redeem,mfd_7])'
utils.save_test(test_y, test_predict, save_path, test_date, lr, run_times,
time_step, rnn_unit, spend, test_score, describe)
def predict_prepare(data_x = None,model_path = None,config_file = 'web_flask/LSTM/config.json'):
config_file = config_file
configs = json.load(open(config_file, 'r'))
'''
data_loader = DataLoader(
os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'],
normalise_meth=configs['data']['normalise']
)
# 用所有数据进行预测
data_x,data_y = data_loader.get_all_data(configs['data']['sequence_length'],\
normalise=configs['data']['normalise'])
data_x = data_x[-1]
'''
model = Model()
model_way = './web_flask/LSTM/saved_models/20052019-174244-e60.h5'
model.load_model(model_way)
predictions = model.predict_point_by_point(data_x)
# print(predictions)
''' 每五分钟预测一个值，貌似是错的
def _parseModelNode(xmlSegments, xmlMaterials, lights, name, node):
"""
parses a model node and creates a model instance.
"""
params = {}
segments = []
sLights = []
for child in list(node):
if child.tag == "segment":
segmentName = child.get('name')
segment = loadXMLSegment(xmlSegments, segmentName)
# load additional attributes into the datatype (rotation,translation,..)
segment = _parseModelSegmentNode(xmlMaterials, lights, segment, child)
segments.append(segment)
elif child.tag == "state":
xmlParseStateParam(child, params)
elif child.tag == "light":
name = child.get('name')
for l in lights:
if l.name==name:
sLights.append(l)
break
else:
print "WARNING: unknown model tag '%s'" % child.tag
m = Model(segments=segments, params=params)
m.setLights(sLights)
return m
def __init__(self, texfilespath='.'):
super().__init__()
self.colors = ColorScheme()
self.arrow = Arrow('midarrow')
self.model = Model()
self.path = os.path.join(texfilespath,
'tex-elements/').replace('\\', '/')
def main(_):
pp.pprint(vars(FLAGS))
sess_config = tf.ConfigProto(
device_count={"CPU": 3},
inter_op_parallelism_threads=0,
intra_op_parallelism_threads=0,
allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
if FLAGS.model == 'mmd':
from core.model import MMD_GAN as Model
elif FLAGS.model == 'gan':
from core.gan import GAN as Model
elif FLAGS.model == 'wgan_gp':
from core.wgan_gp import WGAN_GP as Model
elif FLAGS.model == 'cramer':
from core.cramer import Cramer_GAN as Model
elif FLAGS.model == 'smmd':
from core.smmd import SMMD as Model
elif FLAGS.model == 'swgan':
from core.smmd import SWGAN as Model
else:
raise ValueError("unknown model {}".format(FLAGS.model))
#if FLAGS.multi_gpu:
# from core.model_multi_gpu import MMD_GAN as Model
with tf.Session(config=sess_config) as sess:
#sess = tf_debug.tf_debug.TensorBoardDebugWrapperSession(sess,'localhost:6064')
#sess.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan)
if FLAGS.dataset == 'mnist':
gan = Model(sess, config=FLAGS, batch_size=FLAGS.batch_size, output_size=28, c_dim=1,
data_dir=FLAGS.data_dir)
elif FLAGS.dataset == 'cifar10':
gan = Model(sess, config=FLAGS, batch_size=FLAGS.batch_size, output_size=32, c_dim=3,
data_dir=FLAGS.data_dir)
elif FLAGS.dataset in ['celebA', 'lsun', 'imagenet']:
gan = Model(sess, config=FLAGS, batch_size=FLAGS.batch_size, output_size=FLAGS.output_size, c_dim=3,
data_dir=FLAGS.data_dir)
else:
gan = Model(
sess, batch_size=FLAGS.batch_size,
output_size=FLAGS.output_size, c_dim=FLAGS.c_dim,
data_dir=FLAGS.data_dir)
if FLAGS.is_train:
gan.train()
gan.pre_process_only()
elif FLAGS.print_pca:
gan.print_pca()
elif FLAGS.visualize:
gan.load_checkpoint()
visualize(sess, gan, FLAGS, 2)
else:
gan.get_samples(FLAGS.no_of_samples, layers=[-1])
if FLAGS.log:
sys.stdout = gan.old_stdout
gan.log_file.close()
gan.sess.close()
def setUp(self):
super().setUp()
for i in range(20):
Model.create()
self.paginator = BaseCursorPagination(Model.id,
settings.PAGINATE_BY,
model=Model)
def run():
segmentation_model = Model(50, 50, 0.0009)
get_batches = helper.gen_batch_function()
with tf.Session() as sess:
correct_label = tf.placeholder(
tf.int32, [None, None, None, segmentation_model.no_of_classes],
name='correct_label')
input_image, keep_prob, last_layer = segmentation_model.create_model_graph(
sess, cs.VGG_MODEL, correct_label)
sess.run(tf.global_variables_initializer())
for epoch in range(segmentation_model.no_of_epochs):
print("EPOCH {} ....".format(epoch + 1))
for image, label in get_batches(segmentation_model.batch_size):
res, _, loss = sess.run([
last_layer, segmentation_model.train_op,
segmentation_model.cross_entropy_loss
],
feed_dict={
input_image: image,
correct_label: label,
keep_prob: 0.8
})
print("Loss: = {}".format(loss))
print()
run_inference_on_test(segmentation_model, sess, keep_prob, input_image)
def make_simple_model(self) -> Model:
graph = Graph()
# two inputs
x = Input(
'input',
[1, 5, 5, 3],
Float32(),
)
w = Constant(
'weight',
Float32(),
np.zeros([1, 2, 2, 3]),
dimension_format='NHWC',
)
# Conv
conv = Conv('conv', [1, 4, 4, 1],
Float32(), {
'X': x,
'W': w
},
kernel_shape=[2, 2])
# One output
y = Output('output', [1, 4, 4, 1], Float32(), {'input': conv})
# add ops to the graph
graph.add_op_and_inputs(y)
model = Model()
model.graph = graph
return model
def train_network(configs, dataloader):
# build model
model = Model(configs['data']['input_mode'],
configs['data']['output_mode'])
model.build_model(configs['model'])
# in-memory training
out_seq_len = configs['data']['input_sequence_length'] if configs['data'][
'output_mode'] == "many_to_many" else 1
x_train, y_train = dataloader.get_train_data(
in_seq_len=configs['data']['input_sequence_length'],
out_seq_len=out_seq_len)
x_test, y_test = dataloader.get_test_data(
in_seq_len=configs['data']['input_sequence_length'],
out_seq_len=out_seq_len)
history = model.train(
x_train,
y_train,
x_test,
y_test,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
shuffle=configs['model']['shuffle_training_data'],
allow_early_stop=configs['training']['allow_early_stop'],
)
return model, history
def train(Model):
"""Train the model"""
train_set, target, categoricals = utils.load_data(args.dataset,
args.file_name)
dataset_train = Dataset(dataset=train_set,
categorical_indices=categoricals)
target = dataset_train[target]
dataset_train.drop(target, axis=1, inplace=True)
(
categorical_variables,
non_categorical_variables,
) = dataset_train.get_variables_names()
print("Creating cross products dataset")
cross_products = dataset_train.cross_categorical_dataset()
model = Model(categorical_variables, non_categorical_variables)
model = model.build_model(cross_products)
print("Training model")
model.fit(
[
[
train_set[categorical_variables],
train_set[non_categorical_variables]
],
cross_products,
],
target,
epochs=config.EPOCHS,
validation_split=config.VALIDATION_SPLIT,
)
def gradient_check():
tf.set_random_seed(1231)
np.random.seed(1231)
try:
args = get_args()
config = process_config(args.config)
except:
print("Add a config file using \'--config file_name.json\'")
exit(1)
# set logger
path = os.path.dirname(os.path.abspath(__file__))
path1 = os.path.join(path, 'core/model.py')
path2 = os.path.join(path, 'core/train.py')
logger = get_logger('log', logpath=config.summary_dir+'/',
filepath=os.path.abspath(__file__), package_files=[path1, path2])
logger.info(config)
batch_sizes = [1,4,16,32,64,128,256,512,1024]
precon = False
for bs in batch_sizes:
start_time = time.time()
print("processing batch size {}".format(bs))
# load data
train_loader, test_loader = load_pytorch(config)
# define computational graph
sess = tf.Session()
model_ = Model(config, _INPUT_DIM[config.dataset], len(train_loader.dataset))
trainer = Trainer(sess, model_, train_loader, test_loader, config, logger)
trainer.grad_check(sess, bs, precon)
print('batch size {} takes {} secs to finish'.format(
bs, time.time()-start_time))
tf.reset_default_graph()
precon = True
for bs in batch_sizes:
start_time = time.time()
print("processing batch size {}".format(bs))
# load data
train_loader, test_loader = load_pytorch(config)
# define computational graph
sess = tf.Session()
model_ = Model(config, _INPUT_DIM[config.dataset], len(train_loader.dataset))
trainer = Trainer(sess, model_, train_loader, test_loader, config, logger)
trainer.grad_check(sess, bs, precon)
print('batch size {} takes {} secs to finish'.format(
bs, time.time()-start_time))
tf.reset_default_graph()
def importRoad(self):
model = Model()
model.open(os.path.join(self.tempath, 'model_0.db'), 'w')
importer = RoadImporter(model, self.mesh)
for name in os.listdir(self.roadpath):
if name.endswith('.txd'):
importer.process(os.path.join(self.roadpath, name))
model.close()
def __init__(self, dataset):
# self.imagegen = ImageDataGenerator(shear_range=0.2,zoom_range=0.2,horizontal_flip=True)
self.dataset = dataset
self.train_x,self.train_y = dataset[0]
self.test_x,self.test_y = dataset[1]
self.learning_rate = 0.16
self.eps = 2e-9
self.params = {}
self.model = Model()
def __init__(self, parent=None):
super(ImportWindow, self).__init__(parent)
# Manual import window instance.
self.ui = ManualImportWindow()
# Initialize manual import window.
self.ui.setupUi(self)
# Model instance.
self.mongodb_obj = Model()
def exportAll(self):
builder = AllDataBuilder(self.grid, self.rect)
builder.open(os.path.join(self.dstpath, 'alldata'))
for i in range(0, self.MAX_LEVEL_NUM):
model = Model()
model.open(os.path.join(self.tempath, 'model_%d.db' % (i+1)), 'r')
builder.build(model, i)
model.close()
builder.save()
builder.close()
def buildRoad(self):
for i in range(self.MAX_LEVEL_NUM):
src = Model()
dst = Model()
src.open(os.path.join(self.tempath, 'model_%d.db' % i), 'r')
dst.open(os.path.join(self.tempath, 'model_%d.db' % (i+1)), 'w')
builder = RoadBuilder(src, dst, self.mesh)
builder.process(i)
src.close()
dst.close()
def __init__(self):
Model.__init__(self)
winSize = Config().attr.game.window.size
#Initializing a (i, j) list
lins = int(winSize[0]/16)
cols = int(winSize[1]/16)
Config().attr.game.habitat.size = (lins, cols)
self.gridSize = (lins, cols)
self.keepAlive = []
self.grid = [[None] * cols for i in range(lins)]
def evaluate(args):
G = Model(net=mlp_G(), loss=None, optimizer=None)
model_path = os.path.join(args.output_dir, args.model_name)
print("Loading model from ", model_path)
G.load(model_path)
noise = get_noise(size=(128, args.nz))
samples = G.forward(noise)
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
save_path = os.path.join(args.output_dir, "evaluate.png")
save_batch_as_images(save_path, samples)
def predict(test):
# initialize dataLoader with split of 0
cleaner.main_func()
data = DataLoader(test, 0, configs['data']['columns'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'], normalise=False)
model = Model()
model.load_model('saved_models/tracker.h5')
predictions = model.predict_point_by_point(x_test)
plot_results(predictions, y_test)
return "OK"
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']): os.makedirs(configs['model']['save_dir'])
data = DataLoader(
os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns']
)
model = Model()
model.build_model(configs)
x, y = data.get_train_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
if not configs['training']['train']:
model.load_model(filepath='saved_models/02102019-164727-e2.h5')
else:
model.train(
x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir']
)
# out-of memory generative training
# steps_per_epoch = math.ceil(
# (data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
# model.train_generator(
# data_gen=data.generate_train_batch(
# seq_len=configs['data']['sequence_length'],
# batch_size=configs['training']['batch_size'],
# normalise=configs['data']['normalise']
# ),
# epochs=configs['training']['epochs'],
# batch_size=configs['training']['batch_size'],
# steps_per_epoch=steps_per_epoch,
# save_dir=configs['model']['save_dir']
# )
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
# predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'],
# configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
# plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
predictions = model.predict_point_by_point(x_test)
plot_results(predictions, y_test)
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataProcessor(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=".")
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
predictions_pointbypoint = model.predict_point_by_point(x_test)
plot_results(predictions_pointbypoint, y_test)
predictions_fullseq = model.predict_sequence_full(
x_test, configs['data']['sequence_length'])
plot_results(predictions_fullseq, y_test)
def main(train_after=False):
config_file = 'web_flask/LSTM/config.json'
configs = json.load(open(config_file, 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(configs['data']['filename'],
configs['data']['train_test_split'],
configs['data']['columns'],
normalise_meth=configs['data']['normalise'])
model = Model()
model.build_model(configs) if not train_after else \
model.load_model(os.path.join( configs['model']['save_dir'],configs['model']['model_name']))
history = LossHistory()
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# in-memory training
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
save_dir=configs['model']['save_dir'],
history=history,
x_test=x_test,
y_test=y_test)
'''
# out-of memory generative training
steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
model.train_generator(
data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']
),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir']
)
'''
history.loss_plot('epoch')
#loss, accuracy = model.model.evaluate(x_test, y_test)
#print(loss,accuracy)
#predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
#predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
predictions = model.predict_point_by_point(x[0]) #_test)
#plot_results_multiple(predictions, y, configs['data']['sequence_length'])
plot_results(predictions, y)
def main(model_name=None):
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
if model_name:
saved_model = os.path.join(configs['model']['save_dir'], model_name)
model.load_model(saved_model)
else:
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
'''
# out-of memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
if not model_name:
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir'])
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
def main():
tf.set_random_seed(1231)
np.random.seed(1231)
try:
args = get_args()
config = process_config(args.config)
except:
print("Add a config file using \'--config file_name.json\'")
exit(1)
makedirs(config.summary_dir)
makedirs(config.checkpoint_dir)
# set logger
path = os.path.dirname(os.path.abspath(__file__))
path1 = os.path.join(path, 'core/model.py')
path2 = os.path.join(path, 'core/train.py')
logger = get_logger('log', logpath=config.summary_dir+'/',
filepath=os.path.abspath(__file__), package_files=[path1, path2])
logger.info(config)
# load data
train_loader, test_loader = load_pytorch(config)
# define computational graph
sess = tf.Session()
model_ = Model(config, _INPUT_DIM[config.dataset], len(train_loader.dataset))
trainer = Trainer(sess, model_, train_loader, test_loader, config, logger)
trainer.train()
def main(args):
if args.seed >= 0:
random_seed(args.seed)
# create output directory for saving result images
if not os.path.exists('./output'): os.mkdir('./output')
# define network we are going to load
net = Net([
Conv2D(kernel=[5, 5, 1, 6], stride=[1, 1], padding="SAME"),
ReLU(),
MaxPool2D(pool_size=[2, 2], stride=[2, 2]),
Conv2D(kernel=[5, 5, 6, 16], stride=[1, 1], padding="SAME"),
ReLU(),
MaxPool2D(pool_size=[2, 2], stride=[2, 2]),
Flatten(),
Dense(120),
ReLU(),
Dense(84),
ReLU(),
Dense(10)
])
# load the model
model = Model(net=net, loss=SoftmaxCrossEntropyLoss(), optimizer=Adam())
print('loading pre-trained model file', args.model_path)
model.load(args.model_path)
# create pyplot window for on-the-fly visualization
img = np.ones((1, 28, 28, 1))
fig = disp_mnist_batch(img)
# actual visualization generations
layer_name = 'conv-layer-1'
print('[ ' + layer_name + ' ]')
images = am_visualize_conv_layer(model, 0, fig)
save_batch_as_images('output/{}.png'.format(layer_name),
images,
title='visualized feature maps for ' + layer_name)
layer_name = 'conv-layer-2'
print('[ ' + layer_name + ' ]')
images = am_visualize_conv_layer(model, 3, fig)
save_batch_as_images('output/{}.png'.format(layer_name),
images,
title='visualized feature maps for ' + layer_name)
def __init__(self, parent=None):
super(MainForm, self).__init__(parent)
# Main window instance.
self.ui = Ui_MainWindow()
# Initialize the main window user interface.
self.ui.setupUi(self)
# Model instance.
self.mongodb_obj = Model()
# Initialize in advance.
for i in range(1, 5):
self._init_all(i)
# Video player instance.
self.video_window = VideoWindow()
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']):
os.makedirs(configs['model']['save_dir'])
data = DataLoader(os.path.join('../data', configs['data']['filename']),
os.path.join('../data', configs['data']['VIMfile']),
configs['data']['train_test_split'],
configs['data']['columns'])
model = Model()
model.build_model(configs)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
'''
# Out-of memory generative training
steps_per_epoch = math.ceil(
(data.len_train - configs['data']['sequence_length']) /
configs['training']['batch_size'])
model.train_generator(data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir'])
x_test, y_test, p0_vec = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
#predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
predictions = model.predict_point_by_point(x_test)
pred = predictions.reshape((predictions.size, 1))
#plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
#plot_results(pred, y_test) #normalised predictions
# De-normalise & plot
p_pred, p_true = denorm_transform(p0_vec, pred, y_test)
plot_results(p_pred, p_true) #de-normalised, i.e., original fex units
# Compute evaluation metrics
assess = EvalMetrics(p_true, p_pred)
MAE = assess.get_MAE()
RMSE = assess.get_RMSE()
print("MAE on validation set is: %f" % MAE)
print("RMSE on validation set is: %f" % RMSE)
def predict():
configs = json.load(open(CONFIG, 'r'))
data = DataLoader(DATA, configs['data']['train_test_split'],
configs['data']['columns'])
global model
if model == None:
model = Model()
model.load_model(MODEL)
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
if TYPE == "sequence":
predictions = model.predict_sequences_multiple(
x_test, configs['data']['sequence_length'],
configs['data']['sequence_length'])
plot_results_multiple(predictions, y_test,
configs['data']['sequence_length'])
if TYPE == "point" or TYPE == "predict":
predictions = model.predict_point_by_point(x_test)
if TYPE == "full":
predictions = model.predict_sequence_full(
x_test, configs['data']['sequence_length'])
if TYPE == "full" or TYPE == "point":
plot_results(predictions, y_test)
if TYPE == "predict":
predicted_value = data.denormalize_windows(
predictions[-1], configs['data']['sequence_length'])
sys.stdout.write("--END--{}--END--\n".format(predicted_value))
else:
sys.stdout.write("--END--")
def initialisation():
global beta, GLOBAL_REF_MODEL
dataset_path = '.\\dataset\\*.txt'
models = []
for file_name in glob.glob(dataset_path):
models.append(Corpus(path_to_corpus=file_name).freq_result)
m = Model(*models)
beta = m.beta
GLOBAL_REF_MODEL = m.global_ref_model
async def test_pagination_items_per_page(self):
items_per_page = 2
paginator = BaseCursorPagination(Model.id, items_per_page, model=Model)
paginator.model = Model
queryset = Model.select().order_by(Model.id)
queryset = paginator.first(queryset, None, None)
result = queryset.execute()
self.assertEqual(len(result), items_per_page)
async def test_pagination_first_page(self):
queryset = Model.select().order_by(Model.id)
queryset = self.paginator.first(queryset, None, None)
result = queryset.execute()
ids = [i.id for i in result]
self.assertEqual(len(result), settings.PAGINATE_BY)
self.assertIn(1, ids)
self.assertIn(settings.PAGINATE_BY, ids)
def importBack(self):
modelist = []
for i in range(self.MAX_LEVEL_NUM):
model = Model()
model.open(os.path.join(self.tempath, 'model_%d.db' % (i+1)), 'a')
modelist.append(model)
importer = BackImporter(modelist, self.grid)
# back_0
for name in os.listdir(self.backpath_0):
if name.endswith('.txd'):
importer.process(os.path.join(self.backpath_0, name), 0, 2)
for name in os.listdir(self.backpath_2):
if name.endswith('.txd'):
importer.process(os.path.join(self.backpath_2, name), 2, 4)
for name in os.listdir(self.backpath_4):
if name.endswith('.txd'):
importer.process(os.path.join(self.backpath_4, name), 4, 6)
for model in modelist:
model.close()
def cnn_model():
net = Net([
Conv2D(kernel=[3, 3, 1, 2]),
MaxPool2D(pool_size=[2, 2], stride=[2, 2]),
Conv2D(kernel=[3, 3, 2, 4]),
MaxPool2D(pool_size=[2, 2], stride=[2, 2]),
Flatten(),
Dense(1)
])
return Model(net, loss=MSE(), optimizer=SGD())
def to_dict(self):
self.planProgress = self.get_plan_progress()
self.creatorDisplayName = user.get_user_display_name(self.creator_id)
from contact.contactservice import ContactService
self.taskAssigneeName = ContactService.get_instance().get_contact_name(self.assignee_id)
from task.taskservice import TaskService
self.componentKeys = map(lambda x: x.component_id, TaskService.get_instance().fetch_taskcomponents(self.key()))
self.affectedVersionKeys = map(lambda x: x.version_id, TaskService.get_instance().fetch_taskversions(self.key(), True))
self.fixedVersionKeys = map(lambda x: x.version_id, TaskService.get_instance().fetch_taskversions(self.key(), False))
tdt = Model.to_dict(self)
tdt['storyPoints'] = "%0.1f" % self.story_points if self.story_points is not None else ""
return tdt
def main():
configs = json.load(open('config.json', 'r'))
if not os.path.exists(configs['model']['save_dir']): os.makedirs(configs['model']['save_dir'])
data = DataLoader(
os.path.join('data', configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns']
)
model = Model()
model.build_model(configs)
x, y = data.get_train_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
'''
# in-memory training
model.train(
x,
y,
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
save_dir = configs['model']['save_dir']
)
'''
# out-of memory generative training
steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
model.train_generator(
data_gen=data.generate_train_batch(
seq_len=configs['data']['sequence_length'],
batch_size=configs['training']['batch_size'],
normalise=configs['data']['normalise']
),
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'],
steps_per_epoch=steps_per_epoch,
save_dir=configs['model']['save_dir']
)
x_test, y_test = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
)
predictions = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
# predictions = model.predict_sequence_full(x_test, configs['data']['sequence_length'])
# predictions = model.predict_point_by_point(x_test)
plot_results_multiple(predictions, y_test, configs['data']['sequence_length'])
def setUp(self):
""" Sets up tests and starts mock patchers """
# mock open(file) and os.path.getmtime(file)
patcher_open = patch('builtins.open')
patcher_mtime = patch('os.path.getmtime')
self.patchers = [patcher_open, patcher_mtime]
self.addCleanup(self.cleanUp)
self.mock_open = patcher_open.start()
self.mock_open.return_value.__enter__ = lambda s: s
self.mock_open.return_value.__exit__ = mock.Mock()
self.mock_open.return_value.read.return_value = '<html><title>Battrick - Squad</title></html>'
self.mock_mtime = patcher_mtime.start()
self.test_date = datetime(2015, 1, 1, 12, 0, 10)
self.mock_session = mock.Mock(spec=Session)
self.mock_mtime.return_value = mktime(
self.test_date.timetuple()) + 1e-6 * self.test_date.microsecond
self.model = Model(echo=False)
def __init__(self):
Model.__init__(self)
self.habitat = Habitat()
self.state = Game.STATE_PREPARING
class TestModel(TestCase):
""" Tests the model class. """
def cleanUp(self):
""" Stops all unittest.mock patch objects """
for patcher in self.patchers:
patcher.stop()
def setUp(self):
""" Sets up tests and starts mock patchers """
# mock open(file) and os.path.getmtime(file)
patcher_open = patch('builtins.open')
patcher_mtime = patch('os.path.getmtime')
self.patchers = [patcher_open, patcher_mtime]
self.addCleanup(self.cleanUp)
self.mock_open = patcher_open.start()
self.mock_open.return_value.__enter__ = lambda s: s
self.mock_open.return_value.__exit__ = mock.Mock()
self.mock_open.return_value.read.return_value = '<html><title>Battrick - Squad</title></html>'
self.mock_mtime = patcher_mtime.start()
self.test_date = datetime(2015, 1, 1, 12, 0, 10)
self.mock_session = mock.Mock(spec=Session)
self.mock_mtime.return_value = mktime(
self.test_date.timetuple()) + 1e-6 * self.test_date.microsecond
self.model = Model(echo=False)
@mock.patch('core.model.parse_players')
@mock.patch('sqlalchemy.orm.query.Query.count')
@mock.patch('sqlalchemy.orm.query.Query.first')
@mock.patch('sqlalchemy.orm.session.Session.add')
@mock.patch('core.model.parse_team_id')
def test_import_file_calls_parse_players_if_squad_file(self, mock_parse_team_id, mock_add, mock_first, mock_count, mock_parse_players):
""" Tests that parse_players is called if the HTML file is a Squad file """
html_file = HTMLFile()
html_file.type = PageTypes.Squad.value
html_file.HTML = 'some HTML'
mock_parse_players.return_value = []
mock_parse_team_id.return_value = 3000
team = Team()
mock_first.result = team
self.model.import_file(html_file)
mock_parse_players.assert_called_with(html_file.HTML)
@mock.patch('core.model.parse_pavilion')
@mock.patch('sqlalchemy.orm.query.Query.first')
@mock.patch('core.model.parse_team_id')
@mock.patch('core.PyBatBase.Team.add_ranking')
def test_import_file_calls_parse_pavilion_if_pavilion_file(self, mock_ranking,mock_parse_team_id, mock_first, mock_parse_pavilion):
html_file = HTMLFile()
html_file.type = PageTypes.Pavilion.value
html_file.HTML = 'some HTML'
mock_parse_pavilion.return_value = []
mock_parse_team_id.return_value = 3000
team = Team()
team.rankings.append = mock.Mock()
mock_first.result = team
self.model.import_file(html_file)
assert mock_parse_team_id.called
@mock.patch('core.model.parse_team_id')
@mock.patch('sqlalchemy.orm.session.Session.query')
@mock.patch('sqlalchemy.orm.session.Session.add')
def test_import_file_retrieves_team_from_file(self, mock_add, mock_query, mock_parse_team_id):
test_id = 49800
team = Team()
html_file = HTMLFile()
html_file.HTML = 'Some HTML'
mock_query.return_value.filter_by = mock.Mock()
mock_parse_team_id.return_value = test_id
mock_result = mock.Mock(spec=Query)
mock_result.count = mock.Mock()
mock_result.count.return_value = 1
mock_query.return_value.filter_by.return_value = mock_result
self.model.import_file(html_file)
mock_query.assert_called_with(Team)
assert mock_parse_team_id.called
mock_query.return_value.filter_by.assert_called_with(id=test_id)
@mock.patch('sqlalchemy.orm.session.Session.query')
def test_has_teams_returns_false_if_no_teams(self, mock_query):
mock_query.return_value.count.return_value = 0
result = self.model.has_teams()
assert_false(result)
@mock.patch('sqlalchemy.orm.session.Session.query')
def test_has_teams_returns_true_if_teams(self, mock_query):
mock_query.return_value.count.return_value = 1
result = self.model.has_teams()
assert_true(result)
def to_dict(self):
self.creatorDisplayName = user.get_user_display_name(self.creator_id)
from contact.contactservice import ContactService
self.groupName = ContactService.get_instance().get_group_name(self.group_id)
return Model.to_dict(self)
def to_dict(self):
from contact.contactservice import ContactService
self.contactName = ContactService.get_instance().get_contact_name(self.contact_id)
return Model.to_dict(self)
def to_dict(self):
self.creatorDisplayName = user.get_user_display_name(self.creator_id)
return Model.to_dict(self)
def to_dict(self):
self.bind_user_account = user.get_user(user_id=self.bind_user_id).u_account if self.bind_user_id != None and self.bind_user_id != model.EMPTY_UID else ""
self.creatorDisplayName = user.get_user_display_name(self.creator_id)
from contact.contactservice import ContactService
self.groupKeys = list(set(map(lambda x: x.group_id, ContactService.get_instance().fetch_contactgroups(contact_id=self.key()))))
return Model.to_dict(self)
