def train(self, epoch=10, batch_size=32, gpu=False):
if gpu:
cuda.check_cuda_available()
xp = cuda.cupy if gpu else np
self.batch_size = batch_size
label_types = ['none', 'tap', 'up', 'down', 'right', 'left']
self.model = Alex(len(label_types))
optimizer = optimizers.MomentumSGD(lr=0.01, momentum=0.9)
optimizer.setup(self.model)
if gpu:
self.model.to_gpu()
training_data = TrainingData(IMAGE_ROOT, NOTE_ROOT, VIDEO_ROOT, SONG_LIST_PATH)
self.x_train, self.x_test, self.y_train, self.y_test = training_data.get_train_data(label_types)
data_size = self.x_train.shape[0]
for ep in range(epoch):
print('epoch {0}/{1}: (learning rate={2})'.format(ep + 1, epoch, optimizer.lr))
indexes = np.random.permutation(data_size)
for i in range(0, data_size, self.batch_size):
x_batch = self.x_train[indexes[i:i + self.batch_size]]
y_batch = self.y_train[indexes[i:i + self.batch_size]]
x = chainer.Variable(x_batch)
t = chainer.Variable(y_batch)
optimizer.update(self.model, x, t)
print("loss: {0}".format(self.model.loss.data))
serializers.save_npz(MODEL_PATH, self.model)
optimizer.lr *= 0.97
def create_engine(db_path: str, training_data_path: str):
db = NeuralNetworkDB()
db.open(db_path)
config = Config(db)
training_data = TrainingData(db)
training_data.load_file(training_data_path)
# engine.neural_network_loader = NeuralNetworkLoader(engine)
#
engine = Engine()
engine._db = db
engine._config = config
engine._training_data = training_data
Engine._instance = engine
engine._conductor = Conductor()
engine._evolution_processor = EvolutionProcessor()
def calculate_sales_probality_per_offer(self):
probability_per_offer = []
for joined_market_situations in self.testing_data.joined_data.values():
for jms in joined_market_situations.values():
if self.settings["initial_merchant_id"] in jms.merchants:
for offer_id in jms.merchants[self.settings["initial_merchant_id"]].keys():
features_ps = extract_features(offer_id, TrainingData.create_offer_list(jms), False, self.testing_data.product_prices)
probability = self.ml_engine.predict(jms.merchants[self.settings["initial_merchant_id"]][offer_id].product_id, [features_ps])
probability_per_offer.append((int(offer_id), probability[0]))
write_calculations_to_file(probability_per_offer, self.settings['output_file'])
class CrossValidator:
def __init__(self, settings: dict, ml_engine: MlEngine):
self.settings = settings
self.ml_engine = ml_engine
self.testing_data = TestingData()
self.training_data = TrainingData(self.settings['merchant_token'], self.settings['merchant_id'])
def cross_validation(self):
logging.debug('Creating training set')
self.create_training_data()
logging.debug('Creating testing set')
self.create_testing_data()
logging.debug('Perform learning')
self.perform_learning()
logging.debug('Calculate probabilties per offer and write them to disk')
self.calculate_sales_probality_per_offer()
logging.debug('Finished!')
def create_training_data(self):
self.training_data.append_by_csvs(self.settings['market_situation_csv_path'],
self.settings['buy_offer_csv_path'],
self.settings["initial_merchant_id"])
def create_testing_data(self):
self.testing_data.append_by_csvs(self.settings['testing_set_csv_path'],
self.settings['initial_merchant_id'])
def perform_learning(self):
self.ml_engine.train_model(self.training_data.convert_training_data())
def calculate_sales_probality_per_offer(self):
probability_per_offer = []
for joined_market_situations in self.testing_data.joined_data.values():
for jms in joined_market_situations.values():
if self.settings["initial_merchant_id"] in jms.merchants:
for offer_id in jms.merchants[self.settings["initial_merchant_id"]].keys():
features_ps = extract_features(offer_id, TrainingData.create_offer_list(jms), False, self.testing_data.product_prices)
probability = self.ml_engine.predict(jms.merchants[self.settings["initial_merchant_id"]][offer_id].product_id, [features_ps])
probability_per_offer.append((int(offer_id), probability[0]))
write_calculations_to_file(probability_per_offer, self.settings['output_file'])
def load_data(resource_name, language='en'):
# type: (Text, Optional[Text]) -> TrainingData
"""Load training data from disk.
Merges them if loaded from disk and multiple files are found."""
files = utils.list_files(resource_name)
data_sets = [_load(f, language) for f in files]
data_sets = [ds for ds in data_sets if ds]
if len(data_sets) == 0:
return TrainingData()
elif len(data_sets) == 1:
return data_sets[0]
else:
return data_sets[0].merge(*data_sets[1:])
def main():
# Analyze the dataset paths
print 'Loading datasets...'
ds_train_samples = DataSet(TRAINING_SAMPLES_PATH)
ds_train_gt = DataSet(GROUND_TRUTH_PATH)
print 'Loading data generation parameters...'
data = TrainingData(ds_train_samples, ds_train_gt)
# Load the data format parameters or use the default ones
try:
data.load_config_file(DATA_FORMAT_CONFIG_PATH)
except IOError:
data.config['patch_size'] = DEFAULT_PATCH_SIZE
try:
data.save_config_file(DATA_FORMAT_CONFIG_PATH)
except IOError:
print 'Cannot load data format parameters.'
# Load the data generation parameters or use the default ones
try:
genconf = data.load_gen_config(DATA_GENERATION_CONFIG_PATH)
except IOError:
genconf = data.random_gen_config(DEFAULT_PATCHES_PER_IMG)
try:
data.save_gen_config(DATA_GENERATION_CONFIG_PATH, genconf)
except IOError:
print 'Cannot save generation parameters.'
# Generate the data itself
print 'Generating data...'
samples_data = data.generate_input_data(genconf)
gt_data = data.generate_ground_truth_data(genconf)
print samples_data.shape[0], 'training samples were generated.'
import os
import sys
sys.path.append('./src')
sys.path.append('./src/lib')
from training_data import TrainingData
csv_training_data = os.path.join('csv', 'training_data.csv')
dirname = './json'
training_data = TrainingData(csv_training_data)
training_data.export(dirname)
print("--- Successfully exported JSON training data under {}/ ---".format(
dirname))
def arrange(self):
self.ml_testengine.product_model_dict['1'] = ''
training_data = TrainingData('', '')
training_data.product_prices = {'1': 10.0}
self.tested.training_data = training_data
def __init__(self, settings: dict, ml_engine: MlEngine):
self.settings = settings
self.ml_engine = ml_engine
self.testing_data = TestingData()
self.training_data = TrainingData(self.settings['merchant_token'], self.settings['merchant_id'])
params['contributions'] = [dG0_rc, dG0_gc]
params['covariances'] = [V_rc, V_gc, V_inf]
params['MSEs'] = [MSE_rc, MSE_gc, MSE_inf]
params['projections'] = [P_R_rc,
P_R_gc * G.T * P_N_rc,
P_N_gc * G.T * P_N_rc,
P_R_gc,
P_N_rc,
P_N_gc]
params['inverses'] = [inv_S, inv_GS, inv_SWS, inv_GSWGS]
# Calculate the total of the contributions and covariances
cov_dG0 = V_rc * MSE_rc + V_gc * MSE_gc + V_inf * MSE_inf
return dG0_cc, cov_dG0, params
if __name__ == '__main__':
from kegg_model import KeggModel
reaction_strings = sys.stdin.readlines()
td = TrainingData()
cc = ComponentContribution(td)
model = KeggModel.from_formulas(reaction_strings)
model_dG0, model_cov_dG0 = cc.estimate_kegg_model(model)
model_dG0_prime = model_dG0 + model.get_transform_ddG0(pH=7.5, I=0.2, T=298.15)
sys.stdout.write('[' +
', '.join([str(x) for x in model_dG0.flat]) + '; ' +
', '.join([str(x) for x in model_dG0_prime.flat]) +
']')
def load_image(image_path):
root = os.path.basename(image_path)
return TrainingData(image_path,
measures=measurements,
kind="good" if root.find('good') != -1 else "bad",
compare=False)
#import matplotlib.pyplot as plt
measurements = (
contrast,
noise,
blur,
composition,
#faces, # => under composition
)
max_size = (640, 480
) # set to None to use original image sizes (can take awhile!)
# good - bad_qualities = possible bad qualities that might be marked
california_night = TrainingData(
'good/Another Beautiful California Night.jpg',
measures=measurements,
kind='good',
)
bnw_horse = TrainingData(
'good/AP McCoy Black & White Horse Racing Photo.jpg',
measures=measurements,
kind='good', #bad_qualities=['grayscale'],
)
dreamer = TrainingData(
'good/Beautiful Dreamer, Awake Unto Me.jpg',
measures=measurements,
kind='good', # bad_qualities=['composition']
)
tiger = TrainingData(
'good/Beautiful Face.jpg',
measures=measurements,
def main():
training_data = TrainingData(IMAGE_ROOT, NOTE_ROOT, VIDEO_ROOT, SONG_LIST_PATH)
x_train, x_test, y_train, y_test = training_data.get_train_data(['none', 'tap', 'up', 'down', 'right', 'left'])
# coding=utf-8
from training_data import TrainingData
import json
from random import Random
td = TrainingData(json_str="""
{
"training_set_name": "add",
"inputs": ["input_01", "input_02"],
"outputs": ["add"],
"combo_set_ratio": 0.8,
"combo_set": [],
"training_set": [],
"testing_set": []
}
""")
r = Random()
def get_datum():
while True:
a = r.uniform(0, 10)
b = r.uniform(0, 10)
yield {"in": [a, b], "out": [a+b]}
datum_it = get_datum()
for i in range(1000):
td._training_set.append(datum_it.__next__())