def score_regressor():
from solutions.pos_regressor import PositionRegressor
regressor = PositionRegressor()
data = load_data('./data/regression_data.pkl')
regressor.train(data)
positions = np.asarray([info['agent_pos'] for info in data['info']])
pred_pos = regressor.predict(data['obs'])
mse = np.mean(np.sum(np.square(positions - pred_pos), axis=1))
target_mse = 1.1e-5
if mse < target_mse:
mse = 0.
mse_dist = (mse - target_mse)
score = 1. - mse_dist
score = min(max(0., score), 1.)
test_data = load_data('./data/reg_test_data.pkl')
test_positions = np.asarray(
[info['agent_pos'] for info in test_data['info']])[:10, ]
test_pred_pos = regressor.predict(test_data['obs'][:10])
test_mse = np.mean(
np.sum(np.square(test_positions - test_pred_pos), axis=1))
target_mse = 0.038
if test_mse < target_mse:
test_mse = 0.
test_mse_dist = (mse - target_mse)
test_score = 1. - mse_dist
test_score = min(max(0., score), 1.)
# print('train_mse:', mse)
# print('test_mse:', test_mse)
score = 0.5 * score + 0.5 * test_score
return score
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--train', action='store_true')
parser.add_argument('--test', action='store_true')
parser.add_argument('--model')
args = parser.parse_args()
if args.train and args.model:
nn = model.generate_model()
training_input = data_utils.load_data(training_input_dir)
training_output = data_utils.load_data(training_output_dir)
nn.fit(training_input, training_output, batch_size=128, epochs=50)
model.saveModel(nn, args.model)
test = input("Do you want to test with the test images too? ")
if test == 'yes':
test_input = data_utils.load_data(test_input_dir)
test_output = nn.predict(test_input)
print(test_output.shape)
data_utils.save_images(test_output_dir, test_input_dir,
test_output)
elif args.test and args.model:
nn = model.loadModel(args.model)
test_input = data_utils.load_data(test_input_dir)
test_output = nn.predict(test_input)
print(test_output.shape)
data_utils.save_images(test_output_dir, test_input_dir, test_output)
def main():
word2id=load_data(FLAGS.voc_file)
label2id=load_data(FLAGS.tag_voc)
train_data=data_corpus(FLAGS.src_file,word2id,label2id)
dev_data=data_corpus(FLAGS.src_file_dev,word2id,label2id)
nermodel=NERModel(FLAGS,config,word2id,label2id)
nermodel.build_model()
nermodel.train(train_data,dev_data)
def test(model_name, model_ckpt, dataset_name, data_folder):
# model definition
if model_name == 'lenet':
from model.lenet import LeNet
model = LeNet()
else:
from model.modelzoo import create_model
model, input_size = create_model(model_name, n_classes=120)
model = apply_cuda(model)
# load weights
ckpt = torch.load(model_ckpt)
model.load_state_dict(ckpt['state_dict'])
# data source
batch_size = 200
if dataset_name == 'mnist':
test_loader = load_data('test', batch_size, data_folder, dataset_name)
else:
test_loader = load_data('test', batch_size, data_folder, dataset_name,
input_size)
n_batches_test = len(test_loader)
print('==== test phase ====')
avg_acc = float(0)
model.eval()
images_export, labels_export = None, None
for i, (images, labels) in enumerate(test_loader):
if images_export is None or labels_export is None:
images_export = images.data.numpy()
labels_export = labels.data.numpy()
else:
images_export = np.concatenate(
(images_export, images.data.numpy()), axis=0)
labels_export = np.concatenate(
(labels_export, labels.data.numpy()), axis=0)
images, labels = apply_cuda(images), apply_cuda(labels)
logits = model(images)
_, pred = torch.max(logits.data, 1)
if i == 0:
print(images[0])
print(logits[0], pred[0], labels[0])
bs_ = labels.data.size()[0]
match_count = (pred == labels.data).sum()
accuracy = float(match_count) / float(bs_)
print(
datetime.now(),
'batch {}/{} with shape={}, accuracy={:.4f}'.format(
i + 1, n_batches_test, images.shape, accuracy))
avg_acc += accuracy / float(n_batches_test)
print(datetime.now(), 'test results: acc={:.4f}'.format(avg_acc))
print(
datetime.now(),
'total batch to be exported with shape={}'.format(images_export.shape))
export_test_data_to_numpy(images_export, labels_export, data_folder)
def read_all_data(self, actions, data_dir, one_hot=False):
"""
Loads data for training/testing and normalizes it.
Args
actions: list of strings (actions) to load
seq_length_in: number of frames to use in the burn-in sequence
seq_length_out: number of frames to use in the output sequence
data_dir: directory to load the data from
one_hot: whether to use one-hot encoding per action
Returns
train_set: dictionary with normalized training data
test_set: dictionary with test data
data_mean: d-long vector with the mean of the training data
data_std: d-long vector with the standard dev of the training data
dim_to_ignore: dimensions that are not used becaused stdev is too small
dim_to_use: dimensions that we are actually using in the model
"""
train_subject_ids = [1, 6, 7, 8, 9, 11]
test_subject_ids = [5]
train_set, complete_train = data_utils.load_data(
data_dir, train_subject_ids, actions, one_hot)
test_set, complete_test = data_utils.load_data(data_dir,
test_subject_ids,
actions, one_hot)
# Compute normalization stats
data_mean, data_std, dim_to_ignore, dim_to_use = data_utils.normalization_stats(
complete_train)
# Normalize -- subtract mean, divide by stdev
train_set = data_utils.normalize_data(train_set, data_mean, data_std,
dim_to_use, actions, one_hot)
test_set = data_utils.normalize_data(test_set, data_mean, data_std,
dim_to_use, actions, one_hot)
print("done reading data.")
self.train_set = train_set
self.test_set = test_set
self.data_mean = data_mean
self.data_std = data_std
self.dim_to_ignore = dim_to_ignore
self.dim_to_use = dim_to_use
self.train_keys = list(self.train_set.keys())
def _get_buckets():
# test set
test_buckets = data_utils.load_data("test_ids.enc", "test_ids.dec")
# training set
data_buckets = data_utils.load_data("train_ids.enc", "train_ids.dec")
# Count the number of conversation pairs for each bucket.
train_bucket_sizes = [len(data_buckets[b]) for b in range(len(config.BUCKETS))]
# print("Number of samples in each bucket:\n", train_bucket_sizes)
# Total number of conversation pairs.
train_total_size = sum(train_bucket_sizes)
# list of increasing numbers from 0 to 1 that we"ll use to select a bucket.
train_buckets_scale = [sum(train_bucket_sizes[:i + 1]) / train_total_size
for i in range(len(train_bucket_sizes))]
# print("Bucket scale:\n", train_buckets_scale)
return test_buckets, data_buckets, train_buckets_scale
def main(args):
"""
"""
print("Loading dataset...")
X, y = load_data(args.cw_traces, max_length=args.length, max_instances=105)
y = np.zeros(y.shape)
classes = len(np.unique(y))
X_ow, _ = load_data(args.ow_traces,
max_length=args.length,
fname_pattern=r"(\d+)",
max_instances=10000,
open_world=True)
y_ow = np.ones((X_ow.shape[0], )) * classes
print(X.shape, X_ow.shape, y.shape, y_ow.shape)
X, y = np.concatenate([X, X_ow]), np.concatenate([y, y_ow])
unmon_class = classes
classes += 1
# get split
X_tr, y_tr, X_te, y_te, X_va, y_va = make_split(X, y, 0.8, 0.1)
# consider them as float
X_tr = X_tr.astype('float32')
X_va = X_va.astype('float32')
y_tr = y_tr.astype('float32')
y_va = y_va.astype('float32')
print(X_tr.shape[0], 'training samples')
print(X_va.shape[0], 'validation samples')
# convert class vectors to binary class matrices
y_tr = np_utils.to_categorical(y_tr, classes)
y_va = np_utils.to_categorical(y_va, classes)
# train OW model
model = train_model(X_tr, y_tr, X_va, y_va, classes, args.length,
args.model_path)
# prepare OW testing data
unmon_mask = np.equal(y_te, np.ones((y_te.shape[0], )) * unmon_class)
#y_te = np_utils.to_categorical(y_te, classes)
X_te_unmon = X_te[unmon_mask]
y_te_unmon = y_te[unmon_mask]
X_te_mon = X_te[~unmon_mask]
y_te_mon = y_te[~unmon_mask]
do_ow(model, X_te_mon, y_te_mon, X_te_unmon, y_te_unmon, args.log_file)
def read_all_data(actions=walking_lst,
seq_length_in=50,
seq_length_out=25,
data_dir="./data/h3.6m/dataset",
one_hot=True):
"""
Loads data for training/testing and normalizes it.
Args
actions: list of strings (actions) to load
seq_length_in: number of frames to use in the burn-in sequence
seq_length_out: number of frames to use in the output sequence
data_dir: directory to load the data from
one_hot: whether to use one-hot encoding per action
Returns
train_set: dictionary with normalized training data
test_set: dictionary with test data
data_mean: d-long vector with the mean of the training data
data_std: d-long vector with the standard dev of the training data
dim_to_ignore: dimensions that are not used becaused stdev is too small
dim_to_use: dimensions that we are actually using in the model
"""
# === Read training data ===
print("Reading training data (seq_len_in: {0}, seq_len_out {1}).".format(
seq_length_in, seq_length_out))
train_subject_ids = [1, 6, 7, 8, 9, 11]
test_subject_ids = [5]
train_set, complete_train = data_utils.load_data(data_dir,
train_subject_ids,
actions, one_hot)
test_set, complete_test = data_utils.load_data(data_dir, test_subject_ids,
actions, one_hot)
# Compute normalization stats
data_mean, data_std, dim_to_ignore, dim_to_use = data_utils.normalization_stats(
complete_train)
# Normalize -- subtract mean, divide by stdev
train_set = data_utils.normalize_data(train_set, data_mean, data_std,
dim_to_use, actions, one_hot)
test_set = data_utils.normalize_data(test_set, data_mean, data_std,
dim_to_use, actions, one_hot)
print("done reading data.")
return train_set, test_set, data_mean, data_std, dim_to_ignore, dim_to_use
def main():
collection = sys.argv[1]
source = 'sampled_candidates'
#target = 'annotation_task_input'
data_dict = load_data(collection, source)
property_lemma_dict = dict()
overview_dict_list = []
for property, concept_dict_list in data_dict.items():
#print(property)
final_lemma_dict_list = find_label_certainty(concept_dict_list)
property_lemma_dict[property] = final_lemma_dict_list
#prop_counter[property] = len(final_lemma_dict_list)
overview_dict = dict()
overview_dict['property'] = property
overview_dict['pos'] = len(
[d for d in final_lemma_dict_list if d['label'] == 'pos'])
overview_dict['neg'] = len(
[d for d in final_lemma_dict_list if d['label'] == 'neg'])
overview_dict['pos/neg'] = len(
[d for d in final_lemma_dict_list if '/' in d['label']])
overview_dict['total'] = len(final_lemma_dict_list)
overview_dict_list.append(overview_dict)
candidates_to_file(property_lemma_dict, collection)
overview_to_file(overview_dict_list, collection)
def main():
#This sets up the descriptions for each feature. It is a class defined in data_utils
descriptions = data_utils.load_descriptions()
#This gets all the standardized data into a dataframe. There is a 'fips_code' column, and each data feature has a corresponding column.
#You can find the feature names in Data sources/data_values.csv
#To use a specific column, you use pandas way of calling the feature, i.e. data['CRM250207D']
data = data_utils.load_data()
#pca = PCA(n_components=2)
#Your code here to experiment, test, etc
#print 'Got datastream'
start = time.time()
#This gets a subset of features from the dataframe. SO the user provides a set of features (i'll handle how that gets done)
#For your work, just add or replace elements un the subset_features list below
#this will get you a dataframe that has columns: 'fips_code', 'AGE040205D','BNK010205D', etc
query_feature = ['EAN300205D']
subset_features = query_feature + ['AGE040205D','BNK010205D','BPS030205D','CRM250207D']
rank_features = subset_features[1:]
#i.e. subset_features = ['EAN300205D', 'AGE040205D','BNK010205D','BPS030205D','CRM250207D']
#THis is the function that actually gets you the frame with the features you are focusing on, therefor focus_frame
focus_frame = data_utils.get_features(subset_features,data)
svm = SVMRank(focus_frame, 50)
weights_k = svm.train_top_k()
df = pandas.DataFrame(columns=rank_features)
df.loc[0] = weights_k
print "----------------------------------------------------------"
print "Resulting weights of the ranking features:"
print df.to_string(index=False)
frame_ranks_k = svm.get_full_rank()
print
print "Top 10 ranked counties by the query feature (first column is the fips code, second column is query feature, third column is resulting ranking score):"
print frame_ranks_k.ix[:10].to_string(index=False)
def main(unused_argv):
global colourizer
sess = tf.Session()
print("Setting up model graph...", end='')
colourizer = colourizer.Colourizer()
colourizer.setup()
print("done")
print("Loading saver...", end='')
saver = tf.train.Saver(max_to_keep=5)
ckpt = tf.train.get_checkpoint_state(FLAGS.save_dir)
print("done")
print("Restoring model...", end='')
logging.info('Restoring model from %s', ckpt.model_checkpoint_path)
saver.restore(sess, ckpt.model_checkpoint_path)
print("done")
print("Loading data...", end='')
_, (test_l, test_ab, _) = data_utils.load_data()
print("done")
while True:
print("Choose an option (E.g., Enter '1'):")
print("\t(1) Colourize 5 random novel images")
user_input = raw_input()
if user_input == "1":
display_5_images(sess, colourizer, test_l, test_ab)
def main():
config = Config()
vocab = Vocab(config.dict_file)
dev_q, dev_c, dev_s, dev_spans, dev_s_idx, dev_answerable = load_data(
config.dev_file, vocab, config.debug)
dev_data = list(
zip(dev_q, dev_c, dev_s, dev_s_idx, dev_answerable, dev_spans))
ssnet = SSQANet(config)
ssnet.build_model()
ssnet.restore_session(config.dir_model)
batches = batch_loader(dev_data, config.batch_size, shuffle=False)
acc_history = []
em_history = []
for batch in batches:
batch_q, batch_c, batch_s, batch_s_idx, batch_ans, batch_spans = zip(
*batch)
question_lengths, padded_q = zero_padding(batch_q, level=1)
context_lengths, padded_c = zero_padding(batch_c, level=1)
sequence_lengths, sentence_lengths, padded_s = zero_padding(batch_s,
level=2)
batch_acc, batch_em, batch_loss = ssnet.eval(
padded_q, question_lengths, padded_c, context_lengths, padded_s,
sequence_lengths, sentence_lengths, batch_s_idx, batch_ans,
batch_spans)
acc_history.append(batch_acc)
em_history.append(batch_em)
dev_acc = np.mean(acc_history)
dev_em = np.mean(em_history)
print("classification acc :{}".format(dev_acc))
print("EM :{}".format(dev_em))
def main():
print 'loading train and test datas...'
train, test, _ = data_utils.load_data()
longitude_latitude = data_utils.load_longitude_latitude_data()
print 'train:', train.shape, ', test:', test.shape
train_id = train['id']
train_price_doc = train['price_doc']
train.drop(['id', 'price_doc'], axis=1, inplace=True)
test_id = test['id']
test.drop(['id'], axis=1, inplace=True)
# 合并训练集和测试集
conbined_data = pd.concat([train[test.columns.values], test])
conbined_data.columns = test.columns.values
conbined_data = generate_distance_features(conbined_data, longitude_latitude)
train = conbined_data.iloc[:train.shape[0], :]
test = conbined_data.iloc[train.shape[0]:, :]
train['id'] = train_id
train['price_doc'] = train_price_doc
test['id'] = test_id.values
print 'train:', train.shape, ', test:', test.shape
print("Save data...")
data_utils.save_data(train, test, _)
def prepare_data(data_path, test_size, seed, batch_size):
"""
Prepare the dataloaders for the training. These are also passed to the MI
constructor and unpacked in the MI class to compute the mutual information
Args:
data_path: path to load the dataset from.
test_size: percentage of data to use as test (or number of samples).
seed: rng seed.
batch_size: batch size to use.
Returns:
[type]: [description]
"""
X_train, X_test, y_train, y_test = data_utils.load_data(data_path, test_size, seed)
# Prepare data for pytorch
if batch_size != "full":
train_loader = data_utils.create_dataloader(X_train, y_train, batch_size, seed)
test_loader = data_utils.create_dataloader(X_test, y_test, batch_size, seed)
else:
train_loader = data_utils.create_dataloader(X_train, y_train, len(X_train), seed)
test_loader = data_utils.create_dataloader(X_test, y_test, len(X_test), seed)
# Activitiy loaders
full_X, full_y = np.concatenate((X_train, X_test)), np.concatenate((y_train, y_test))
act_full_loader = data_utils.create_dataloader(full_X, full_y, len(full_X), seed, shuffle=False)
return train_loader, test_loader, act_full_loader
def model_01():
"""
猫图像识别，两层网络结构
:return:
"""
X_train, Y_train, X_test, Y_test, classes = load_data() # 猫图像数据
# 模型参数设置
layers_dims = [X_train.shape[0], 7, 1]
num_iter = 2500
learning_rate = 0.0075
print_cost = True
initialization = "sqrt_n"
parameters, costs = basic_model(X_train,
Y_train,
layers_dims=layers_dims,
num_iter=num_iter,
lr=learning_rate,
print_cost=print_cost,
initialization=initialization)
# 预测及评估
prediction_train = predict(parameters, X_train)
prediction_test = predict(parameters, X_test)
print("Train准确率: {}".format(evaluate(prediction_train, Y_train)))
print("test准确率: {}".format(evaluate(prediction_test, Y_test)))
costs_draw(costs, learning_rate=learning_rate)
def train_classification(dataset_params, agent_params, training_params):
dataset = load_data(dataset_params['data_path'])
trainloader = return_trainloader(dataset['trainX'],
dataset['trainY'],
category='classification')
valloader = return_trainloader(dataset['valX'],
dataset['valY'],
category='classification')
data_dict = {
'trainloader': trainloader,
'valloader': valloader
# 'y_handtype_indexes':y_handtype_indexes
}
if dataset_params['datatype'] == f'{dt.DataTypes.HANDRANKSFIVE}':
category_weights = generate_category_weights()
data_dict['category_weights'] = category_weights
print('Data shapes', dataset['trainX'].shape, dataset['trainY'].shape,
dataset['valX'].shape, dataset['valY'].shape)
# dataset['trainY'] = dataset['trainY'].long()
# dataset['valY'] = dataset['valY'].long()
# target = dt.Globals.TARGET_SET[dataset_params['datatype']]
# y_handtype_indexes = return_ylabel_dict(dataset['valX'],dataset['valY'],target)
# print('Target values',np.unique(dataset['trainY'],return_counts=True),np.unique(dataset['valY'],return_counts=True))
train_network(data_dict, agent_params, training_params)
def main():
print 'loading train and test datas...'
train, test, _ = data_utils.load_data()
print 'train:', train.shape, ', test:', test.shape
train_price_doc = train['price_doc']
train.drop(['price_doc'], axis=1, inplace=True)
# 合并训练集和测试集
conbined_data = pd.concat([train[test.columns.values], test])
conbined_data.columns = test.columns.values
conbined_data.index = range(conbined_data.shape[0])
# 时间窗大小
timewindow_days = [30 * 6, 30 * 4, 30 * 2, 30, 20, 10]
conbined_data = perform_time_window(conbined_data, timewindow_days)
conbined_data = perform_groupby_time_window(conbined_data, timewindow_days)
train = conbined_data.iloc[:train.shape[0], :]
test = conbined_data.iloc[train.shape[0]:, :]
train['price_doc'] = train_price_doc
print 'train:', train.shape, ', test:', test.shape
print("Save data...")
data_utils.save_data(train, test, _)
def load_dataset(arg, data_path, in_vocab, slot_vocab, intent_vocab):
"""Returns the dataset that is loaded from the disk.
Args:
arg: The output of the parser.
data_path: The path of the dataset to be loaded.
in_vocab: The vocabulary of the input sentences.
slot_vocab: The vocabulary of slot labels.
intent_vocab: The vocabulary of intent labels.
Returns:
The input data, slot data and the intent data as numpy arrays.
"""
full_path = os.path.join('./data', arg.dataset, data_path)
input_path = os.path.join(full_path, arg.input_file)
slot_path = os.path.join(full_path, arg.slot_file)
intent_path = os.path.join(full_path, arg.intent_file)
in_data, slot_data, intent_data = load_data(input_path, slot_path,
intent_path, in_vocab,
slot_vocab, intent_vocab,
arg.max_seq_len)
return in_data, slot_data, intent_data
def main(args):
# hyper param
root = args.root_dir
assert os.path.exists(root)
tokenizer = AutoTokenizer.from_pretrained(args.model,
cache_dir=args.transformer_cache)
mt_dnn_root = os.path.join(root, args.model)
if not os.path.isdir(mt_dnn_root):
os.makedirs(mt_dnn_root)
task_defs = TaskDefs(args.task_def)
for task in task_defs.get_task_names():
task_def = task_defs.get_task_def(task)
logger.info("Task %s" % task)
for split_name in task_def.split_names:
file_path = os.path.join(root, "%s_%s.tsv" % (task, split_name))
if not os.path.exists(file_path):
logger.warning("File %s doesnot exit")
sys.exit(1)
rows = load_data(file_path, task_def)
dump_path = os.path.join(mt_dnn_root,
"%s_%s.json" % (task, split_name))
logger.info(dump_path)
build_data(
rows,
dump_path,
tokenizer,
task_def.data_type,
lab_dict=task_def.label_vocab,
workers=args.workers,
)
def plot_data():
"""Plot prices and items over time"""
# Load data
prices,items,_ = load_data()
# Price trends for different time periods
prices['year'] = prices.ts.dt.year
prices['month'] = prices.ts.dt.month
prices['dayofweek'] = prices.ts.dt.dayofweek
prices['hour'] = prices.ts.dt.hour
prices.groupby('year',as_index=False)['price'].aggregate({"mean_price":np.mean,
"median_price":np.median}).plot(kind='line', x="year")
prices.groupby('month',as_index=False)['price'].aggregate({"mean_price":np.mean,
"median_price":np.median}).plot(kind='line', x="month")
prices.groupby('dayofweek',as_index=False)['price'].aggregate({"mean_price":np.mean,
"median_price":np.median}).plot(kind='line', x="dayofweek")
prices.groupby('hour',as_index=False)['price'].aggregate({"mean_price":np.mean,
"median_price":np.median}).plot(kind='line', x="hour")
sns.FacetGrid(prices.groupby(["dayofweek","hour"])["price"] \
.aggregate({"mean_price":np.mean, "median_price":np.median}).reset_index(), row="dayofweek") \
.map(plt.plot, "hour", "median_price") \
.set(xlim=(0, 23))
# Grades
item_order = items.loc[items.consumption.argsort()[::-1], 'item']
sns.factorplot(x="item", y="consumption", data=items, kind="bar", order=item_order)
def main():
print 'loading train and test datas...'
train, test, _ = data_utils.load_data()
print 'train:', train.shape, ', test:', test.shape
train_price_doc = train['price_doc']
num_columns = train.select_dtypes(exclude=['object']).columns.values
num_columns = num_columns.tolist()
num_columns.remove('id')
num_columns.remove('timestamp')
print 'perform feature selection in %d numerical features...' % train[num_columns].shape[1]
keep_features = feature_select(train[num_columns], keep_top=0.98)
print 'after feature selection numerical features', len(keep_features)
keep_features.append('id')
keep_features.append('timestamp')
train = train[keep_features]
test = test[keep_features]
train['price_doc'] = train_price_doc
print 'train:', train.shape, ', test:', test.shape
print("Save data...")
data_utils.save_data(train, test, _)
def generate():
random.seed(SEED)
np.random.seed(SEED)
vocab_dict, vocab_res = data_utils.load_vocab('./vocab.txt')
data = data_utils.load_data('data.pkl')
vocab_size = len(vocab_dict)
SEQ_LENGTH = data.shape[1]
generator = Generator(vocab_size, BATCH_SIZE, EMB_DIM, HIDDEN_DIM,
SEQ_LENGTH, START_TOKEN)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
sess.run(tf.global_variables_initializer())
samples = generator.generate(sess)
for i in range(int(1)):
if i > len(samples):
break
arr = samples[i]
poem = ''
for index in arr:
if index != data_utils.EOS_ID:
poem += vocab_res[index]
print(poem)
def main():
args, settings = parse_args_and_settings()
logger = output_utils.Logger(args)
logger.shout('python main.py ' + ' '.join(sys.argv[1:]))
num_threads = 5
torch.set_num_threads(num_threads)
# Load entity map (needed for all phases; primarily for loading data):
entity_idx_to_name, entity_name_to_idx = data_utils.load_entity_map(settings.data.entity_map)
if args.phase == 'train' or args.phase == 'deploy':
# Loading train data is needed for train, but also for deploy, namely if vocabulary doesn't exist yet:
train_data = None
if args.phase == 'train' or not os.path.exists(settings.data.vocabulary):
train_data = data_utils.load_data(settings.data.dataset, entity_name_to_idx, with_keys=True, logger=logger)
# Load vocabulary (and extract from train_data if vocabulary doesn't exist yet)
vocabulary_idx_to_word, vocabulary_word_to_idx = data_utils.get_vocabulary(settings.data.vocabulary,
extract_from=train_data,
logger=logger)
# Avoid loading/generating google news embeddings in deploy phase:
if args.phase == 'deploy' and settings.model.token_emb == config_utils.data_paths["embeddings"]["google_news"]:
settings.model.token_emb = 300
# Appropriate embeddings will be loaded anyway from saved .pt model file.
# TODO: This won't generalize when using other embeddings.
# Load embeddings if needed:
if isinstance(settings.model.token_emb, str):
settings.model.token_emb = embedding_loader.load_word_embeddings(settings.model.token_emb,
settings.data.dataset, train_data, logger)
if isinstance(settings.model.speaker_emb, str):
settings.model.speaker_emb = embedding_loader.load_entity_embeddings(settings.model.speaker_emb,
settings.data.entity_map, logger)
# convenient to compute and store some dependent parameters:
settings.model.vocabulary_size = len(vocabulary_idx_to_word)
settings.model.num_entities = len(entity_idx_to_name)
if args.phase == 'train':
logger.save_config(settings.orig)
logger.say(output_utils.bcolors.BOLD + 'Training on ' + settings.data.dataset)
run_training(settings, train_data, vocabulary_idx_to_word, vocabulary_word_to_idx, logger, not args.no_cuda)
if args.phase == 'deploy':
logger.say(output_utils.bcolors.BOLD + 'Deploying ' + str(len(args.model)) + ' models (' + (
args.run_name if len(args.model) > 1 else args.model[0]) + ')...\n ...on ' + ('folds of ' if not args.no_cv else '') + args.deploy_data)
args.answer_file, with_keys = run_deploy(args.model, settings, args.deploy_data, vocabulary_idx_to_word, vocabulary_word_to_idx, entity_name_to_idx, args.answers_per_fold, args.no_cv, logger, not args.no_cuda)
# After deploying, evaluate (unless not desired or data does not contain reference keys):
if not args.no_eval:
if with_keys is True:
args.phase = 'evaluate'
else:
logger.shout('Warning: Model predictions will not be evaluated, since given data does not contain reference labels. ')
if args.phase == 'evaluate':
logger.say(output_utils.bcolors.BOLD + 'Evaluating ' + ('(not SemEval style) ' if args.no_semeval else '(SemEval style) ') + 'predictions of ' + args.answer_file)
run_evaluate(args.answer_file, args.deploy_data, entity_name_to_idx, entity_idx_to_name, args.no_semeval, logger)
def run_model(model):
'''Solve optimization problem using MedianForecaster -model for forecasting.'''
# Load data
prices, items, schedule = load_data()
# Split to train and test sets based on realized schedule
X_train, y_train, X_test, y_test = split_prices(prices)
# Fit and predict with model
y_pred = model.fit(X_train, y_train).predict(X_test)
# Evaluate model
mse = MSE(y_test, y_pred)
print('MSE: %f' % mse)
# Solve optimization problem with forecasted electricity prices
df_pred = format_forecast_results(X_test, y_pred, prices, schedule)
df_pred.plot()
itemblocks_to_produce,blocks_available,forecasted_block_prices, \
actual_block_prices,item_consumptions,block_order = prepare_optimization(items,schedule,df_pred)
prob = create_optimization_problem(itemblocks_to_produce,
blocks_available,
forecasted_block_prices,
item_consumptions,
name=model.name)
solution_schedule = solve_problem(prob, actual_block_prices,
item_consumptions, block_order)
plot_forecasted_schedule(solution_schedule,
title='Schedule using %s' % model.name)
def main():
collection = sys.argv[1]
#target_property = sys.argv[2]
source = 'data_extracted_sorted'
target = 'data_for_concept_selection_cosine_centroid'
dsm_path = '../../../Data/dsm/wikipedia_full/sgns_pinit1/sgns_pinit1/sgns_rand_pinit1'
model = load_model(dsm_path, 'sgns')
model_creation_type, matrix, wi_dict, dsm_vocab = model
data_dict = load_data(collection, source)
extended_data_dict = dict()
overview_dict_list = []
for property, concept_dict_list in data_dict.items():
#if property == target_property:
print(property)
clean_concept_dict_list = remove_oov(dsm_vocab, concept_dict_list)
new_concept_dict_list = []
n = 200
# get neighbors of positive concept centroid:
concepts_pos = [cd['concept'] for cd in clean_concept_dict_list if cd['label'] == 'pos']
concepts = [cd['concept'] for cd in clean_concept_dict_list]
neighbors, centroid = get_get_candidate_neighbors(model, concepts_pos, n = n)
concept_cosine_dict = get_cosines_to_centroid(model, concepts, centroid)
#print(len(concept_cosine_dict.keys()))
# get cosines to centroid of all concepts
# use nltk pos tagging to select nouns only:
if neighbors:
neighbors_clean = [(c, n) for c, n in neighbors if n not in concepts]
#neighbors_nouns = select_nouns_pos(neighbors_clean)
for cosine, neighbor in neighbors_clean:
neighbor_dict = dict()
neighbor_dict['concept'] = neighbor
neighbor_dict['label'] = 'neg/pos'
neighbor_dict['certainty'] = 'not_certain'
neighbor_dict['sources_str'] = 'wikipedia_sgns_model'
neighbor_dict['categories_str'] = 'neighbors'+'-'+str(n)
neighbor_dict['cosine_centroid'] = str(cosine)
new_concept_dict_list.append(neighbor_dict)
for cd in clean_concept_dict_list:
cos = concept_cosine_dict[cd['concept']]
cd['cosine_centroid'] = str(cos)
print(cd['concept'], cd['cosine_centroid'])
new_concept_dict_list.append(cd)
sorted_concept_dict_list = sorted([(cd['cosine_centroid'], cd) for cd \
in new_concept_dict_list if cd['cosine_centroid'] != '-'])
if sorted_concept_dict_list:
extended_data_dict[property] = [cd for cosine, cd in sorted_concept_dict_list]
print(property, len(extended_data_dict[property]))
else:
print('no concepts in the model vocab: ', property)
data_to_file(collection, extended_data_dict, target)
def train_model(args):
"""Load the data, train the model, test the model, export / save the model
"""
torch.manual_seed(args.seed)
# Open our dataset
train_loader, test_loader = data_utils.load_data(args.test_split,
args.batch_size)
# Create the model
net = model.SonarDNN().double()
optimizer = optim.SGD(net.parameters(), lr=args.lr,
momentum=args.momentum, nesterov=False)
# Train / Test the model
for epoch in range(1, args.epochs + 1):
train(net, train_loader, optimizer, epoch)
test(net, test_loader)
# Export the trained model
torch.save(net.state_dict(), args.model_name)
if args.model_dir:
# Save the model to GCS
data_utils.save_model(args.model_dir, args.model_name)
def test(batch_size):
# Note the x(blur) in the second, the y(full) in the first
y_test, x_test = data_utils.load_data(data_type='test')
g = generator_model()
g.load_weights('weight/generator_weights.h5')
generated_images = g.predict(x=x_test, batch_size=batch_size)
data_utils.generate_image(y_test, x_test, generated_images, 'result/finally/')
def train_model(args):
"""Load the data, train the model, test the model, export / save the model
"""
torch.manual_seed(args.seed)
# Open our dataset
train_loader, test_loader = data_utils.load_data(args.test_split,
args.batch_size)
# Create the model
net = model.SonarDNN().double()
optimizer = optim.SGD(net.parameters(),
lr=args.lr,
momentum=args.momentum,
nesterov=False)
# Train / Test the model
for epoch in range(1, args.epochs + 1):
train(net, train_loader, optimizer, epoch)
test(net, test_loader)
# Export the trained model
torch.save(net.state_dict(), args.model_name)
if args.model_dir:
# Save the model to GCS
data_utils.save_model(args.model_dir, args.model_name)
def run_single_experiment(cfg, diffinit, seed, replace_index):
t0 = time()
# how we convert the cfg into a path and such is defined in ExperimentIdentifier
exp = ExperimentIdentifier(seed=seed,
replace_index=replace_index,
diffinit=diffinit)
exp.init_from_cfg(cfg)
exp.ensure_directory_exists(verbose=True)
path_stub = exp.path_stub()
print('Running experiment with path', path_stub)
# load data
x_train, y_train, x_vali, y_vali, x_test, y_test = load_data(
options=cfg['data'], replace_index=replace_index)
# define model
init_path = get_model_init_path(cfg, diffinit)
model = build_model(**cfg['model'], init_path=init_path)
# prep model for training
prep_for_training(
model,
seed=seed,
optimizer_settings=cfg['training']['optimization_algorithm'],
task_type=cfg['model']['task_type'])
# now train
train_model(model,
cfg['training'],
cfg['logging'],
x_train,
y_train,
x_vali,
y_vali,
path_stub=path_stub)
# clean up
del model
clear_session()
print('Finished after', time() - t0, 'seconds')
def main():
print 'loading train and test datas...'
train, test, _ = data_utils.load_data()
print 'train:', train.shape, ', test:', test.shape
train_id = train['id']
train_price_doc = train['price_doc']
train.drop(['id', 'price_doc'], axis=1, inplace=True)
test_id = test['id']
test.drop(['id'], axis=1, inplace=True)
# 合并训练集和测试集
conbined_data = pd.concat([train[test.columns.values], test])
conbined_data.columns = test.columns.values
# conbined_data = feature_distribute_scale(conbined_data)
conbined_data = feature_discretization(conbined_data)
train = conbined_data.iloc[:train.shape[0], :]
test = conbined_data.iloc[train.shape[0]:, :]
train['id'] = train_id
train['price_doc'] = train_price_doc
test['id'] = test_id
print 'train:', train.shape, ', test:', test.shape
print("Save data...")
data_utils.save_data(train, test, _)
def main():
print 'loading train and test datas...'
train, test, _ = data_utils.load_data()
print 'train:', train.shape, ', test:', test.shape
train_id = train['id']
train_price_doc = train['price_doc']
train.drop(['id', 'price_doc'], axis=1, inplace=True)
test_id = test['id']
test.drop(['id'], axis=1, inplace=True)
# 合并训练集和测试集
conbined_data = pd.concat([train[test.columns.values], test])
conbined_data.columns = test.columns.values
conbined_data = gen_area_features(conbined_data)
conbined_data = gen_school_features(conbined_data)
conbined_data = generate_hospital_features(conbined_data)
conbined_data = generate_population_features(conbined_data)
conbined_data = generate_population_age_features(conbined_data)
conbined_data = generate_build_features(conbined_data)
train = conbined_data.iloc[:train.shape[0], :]
test = conbined_data.iloc[train.shape[0]:, :]
train['id'] = train_id
train['price_doc'] = train_price_doc
test['id'] = test_id
print 'train:', train.shape, ', test:', test.shape
print("Save data...")
data_utils.save_data(train, test, _)
def load_data(dataset_directory , dataset_name):
print "Loading datasets ..."
import os
repo = os.environ.get('MLPYTHON_DATASET_REPO')
if repo is None:
raise ValueError('environment variable MLPYTHON_DATASET_REPO is not defined')
dataset_dir = os.path.join(os.environ.get('MLPYTHON_DATASET_REPO') + '/' + dataset_directory, dataset_name)
input_size = 6
spatial_dimensions = 1
all_data = data_utils.load_data(dir_path=dataset_dir, input_size=input_size, train_filename=None, test_filename=None, background_filename=None,load_to_memory=False)
train_data, train_metadata = all_data['train']
valid_data, valid_metadata = all_data['valid']
finaltrain_data, finaltrain_metadata = all_data['finaltrain']
test_data, test_metadata = all_data['test']
lbl = np.array([int(data[1]) for data in test_data])
def reduce_dimensionality(mlproblem_data, mlproblem_metadata):
mlproblem_metadata['input_size'] = 3 # we need to change the input size from 6 to 3.
return [mlproblem_data[0][:3] , mlproblem_data[1]]
if spatial_dimensions ==1:
import mlpython.mlproblems.classification as mlpb
trainset = mlpb.ClassificationProblem(train_data, train_metadata)
validset = trainset.apply_on(valid_data,valid_metadata)
finaltrainset = trainset.apply_on(finaltrain_data,finaltrain_metadata)
testset = trainset.apply_on(test_data,test_metadata)
elif spatial_dimensions ==0:
import mlpython.mlproblems.generic as mlpg
trainset = mlpg.PreprocessedProblem(data = train_data , metadata = train_metadata , preprocess = reduce_dimensionality)
validset = trainset.apply_on(valid_data, valid_metadata)
testset = trainset.apply_on(test_data, test_metadata)
finaltrainset = trainset.apply_on(finaltrain_data, finaltrain_metadata)
import mlpython.mlproblems.classification as mlpb
trainset = mlpb.ClassificationProblem(trainset, trainset.metadata)
validset = trainset.apply_on(validset,validset.metadata)
finaltrainset = trainset.apply_on(finaltrainset,finaltrainset.metadata)
testset = trainset.apply_on(testset,testset.metadata)
return {'finaltrainset':finaltrainset, 'testset':testset ,'ground_truth':lbl}
def train_model(args):
"""Load the data, train the model, test the model, export / save the model
"""
torch.manual_seed(args.seed)
# Open our dataset
train_loader, test_loader = data_utils.load_data(
args.test_split, args.seed, args.batch_size)
# Create the model
net = model.SonarDNN().double()
optimizer = optim.SGD(net.parameters(), lr=args.lr,
momentum=args.momentum, nesterov=False)
# Train / Test the model
latest_accuracy = 0.0
for epoch in range(1, args.epochs + 1):
train(net, train_loader, optimizer)
latest_accuracy = test(net, test_loader)
# The default name of the metric is training/hptuning/metric.
# We recommend that you assign a custom name. The only functional
# difference is that if you use a custom name, you must set the
# hyperparameterMetricTag value in the HyperparameterSpec object in your
# job request to match your chosen name.
# https://cloud.google.com/ml-engine/reference/rest/v1/projects.jobs#HyperparameterSpec
hpt = hypertune.HyperTune()
hpt.report_hyperparameter_tuning_metric(
hyperparameter_metric_tag='my_accuracy_tag',
metric_value=latest_accuracy,
global_step=args.epochs)
# Export the trained model
torch.save(net.state_dict(), args.model_name)
if args.job_dir:
# Save the model to GCS
data_utils.save_model(args.job_dir, args.model_name)
else:
print('Accuracy: {:.0f}%'.format(latest_accuracy))
def __init__(self,
alpha,
batch_size,
n_epochs,
wordVecLen,
flag_dropout,
datapath,
random_seed,
dropoutRates,
optimizer,
dispFreq,
beam_size,
flag_random_lookup_table,
flag_toy_data,
size_hidden_layer,
dataset,
result_path,
sentence_modeling,
CNN_filter_length,
LSTM_go_backwards
):
model_options = locals().copy()
model_options['rng'] = np.random.RandomState(random_seed)
print 'Loading data'
src_train,src_valid,src_test,dic_w2idx, dic_idx2w, dic_w2embed, dic_idx2embed, embedding = load_data(path=datapath)
if flag_toy_data == True:
src_valid = src_valid[:10]
src_test = src_test[:10]
#src_train = copy.copy(src_valid)
src_train = src_train[:10]
elif flag_toy_data != False:
valid_l = len(src_valid) * flag_toy_data
test_l = len(src_test) * flag_toy_data
train_l = len(src_train) * flag_toy_data
src_valid = src_valid[:int(valid_l)]
src_test = src_test[:int(test_l)]
src_train = src_train[:int(train_l)]
train,pairdict_train = prepare_data(src_train)
valid,pairdict_valid = prepare_data(src_valid)
test,pairdict_test = prepare_data(src_test)
model_options['embedding'] = embedding
(sentence1,sentence1_mask,sentence2,sentence2_mask,y,cost,f_pred,tparams,f_debug) = build_model(model_options)
#f_cost = theano.function([sentence1,sentence1_mask,sentence2,sentence2_mask,y], cost, name='f_cost')
#grads = tensor.grad(theano.gradient.grad_clip(cost, -2.0, 2.0), wrt=tparams.values())
grads = tensor.grad(theano.gradient.grad_clip(cost, -2.0, 2.0), wrt=tparams)
# grads = tensor.grad(cost, wrt=tparams.values())
#f_grad = theano.function([sentence1,sentence1_mask,sentence2,sentence2_mask,y], grads, name='f_grad')
lr = tensor.scalar(name='lr')
if model_options['optimizer'] == 'sgd': optimizer = sgd
elif model_options['optimizer'] == 'rmsprop': optimizer = rmsprop
else: optimizer = adadelta
f_grad_shared, f_update = optimizer(lr, tparams, grads, sentence1,sentence1_mask,sentence2,sentence2_mask,y, cost)
print 'Optimization'
kf_valid = get_minibatches_idx(len(valid), model_options['batch_size'])
kf_test = get_minibatches_idx(len(test), model_options['batch_size'])
print "%d train examples" % len(train)
print "%d valid examples" % len(valid)
print "%d test examples" % len(test)
sys.stdout.flush()
best_validation_score = -np.inf
best_iter = 0
uidx = 0 # the number of update done
for epoch in xrange(model_options['n_epochs']):
print ('Training on %d epoch' % epoch)
sys.stdout.flush()
kf = get_minibatches_idx(len(train), batch_size, shuffle=True)
start_time = time.time()
samples_seen = 0
for _, train_index in kf:
uidx += 1
batch_samples = [train[t] for t in train_index]
samples_seen += len(batch_samples)
#print batch_samples
sentence1,sentence1_mask,sentence2,sentence2_mask,y = data_padding(batch_samples)
#print sentence1,sentence1_mask,sentence2,sentence2_mask,y
#print sentence1.shape,sentence1_mask.shape,sentence2.shape,sentence2_mask.shape,y.shape
#o = f_debug(sentence1,sentence1_mask,sentence2,sentence2_mask,y)
#print o
#print o[0].shape,o[1].shape,o[2].shape,o[3].shape
cost = f_grad_shared(sentence1,sentence1_mask,sentence2,sentence2_mask,y)
f_update(model_options['alpha'])
if np.isnan(cost) or np.isinf(cost):
print 'NaN detected'
return 1., 1., 1.
if np.mod(uidx, dispFreq) == 0:
print 'Epoch ', epoch, 'Update ', uidx, 'Cost ', cost, 'Samples_seen ', samples_seen
sys.stdout.flush()
print 'Epoch ', epoch, 'Update ', uidx, 'Cost ', cost, 'Samples_seen ', samples_seen
sys.stdout.flush()
'''
if epoch % 5 == 0:
kf_train = get_minibatches_idx(len(train), batch_size)
print ('Train_score:')
self.eva(f_pred, src_train, train, pairdict_train, kf_train, model_options)
sys.stdout.flush()
'''
print ('Valid_score:')
top1_res = self.eva(f_pred, src_valid, valid, pairdict_valid, kf_valid, model_options)
self.save_result(model_options['result_path'] + 'dev.on.' + str(epoch) +'th_epoch_' + model_options['dataset'],top1_res)
sys.stdout.flush()
print ('Test_score:')
top1_res = self.eva(f_pred, src_test, test, pairdict_test, kf_test, model_options)
self.save_result(model_options['result_path'] + 'test.on.' + str(epoch) +'th_epoch_' + model_options['dataset'],top1_res)
sys.stdout.flush()
print ('%d epoch completed.' % epoch)
sys.stdout.flush()
'''
if(best_validation_score < valid_score):
best_iter = epoch
best_validation_score = valid_score
print ('Current best_dev_F is %.2f, at %d epoch'%(best_validation_score,best_iter))
'''
end_time = time.time()
minu = int((end_time - start_time)/60)
sec = (end_time - start_time) - 60 * minu
print ('Time: %d min %.2f sec' % (minu, sec))
sys.stdout.flush()
print('Training completed!')
sys.stdout.flush()
paragraph = np.asarray(paragraph) - np.min(paragraph)
paragraph = list(paragraph)
for sentence in paragraph:
fw.write(str(sentence))
fw.write(' ')
fw.write('#')
for category in cur_categories:
fw.write(category)
fw.write(' ')
fw.write('\n')
fw.close()
dataset = 'cs'
datapath = '../data/%s.pkl.gz'%dataset
src_train,src_valid,src_test,dic_w2idx, dic_idx2w, dic_w2embed, dic_idx2embed, embedding = load_data(path=datapath)
res_order = []
res_eva = []
for paragraph, cur_categories in src_test:
n = len(paragraph)
candidates = [x for x in xrange(n)]
guess_order = []
for i in xrange(n):
idx = np.random.randint(n - i)
guess_order.append(candidates[idx])
candidates.remove(candidates[idx])
res_order.append((guess_order, cur_categories))
patial_correct, total_correct = score_rank(guess_order)
res_eva.append(np.asarray([patial_correct, total_correct]))
res_eva = np.asarray(res_eva)
str_ParamOptionValue = ""
dataset_dir = None
if dataset_dir is None:
# Try to find dataset in MLPYTHON_DATASET_REPO
import os
repo = os.environ.get('MLPYTHON_DATASET_REPO')
if repo is None:
raise ValueError('environment variable MLPYTHON_DATASET_REPO is not defined')
dataset_dir = os.path.join(os.environ.get('MLPYTHON_DATASET_REPO') + '/' + dataset_directory, dataset_name)
# Load data
start_time = time.clock()
all_data = data_utils.load_data(dir_path=dataset_dir, input_size=input_size, train_filename=train_filename, test_filename=test_filename, background_filename=background_filename,load_to_memory=False)
train_data, train_metadata = all_data['train']
valid_data, valid_metadata = all_data['valid']
finaltrain_data, finaltrain_metadata = all_data['finaltrain']
test_data, test_metadata = all_data['test']
import mlpython.mlproblems.classification as mlpb
trainset = mlpb.ClassificationProblem(train_data, train_metadata)
validset = trainset.apply_on(valid_data,valid_metadata)
finaltrainset = trainset.apply_on(finaltrain_data,finaltrain_metadata)
testset = trainset.apply_on(test_data,test_metadata)
def compute_error_mean_and_sterror(costs):
classif_errors = np.hstack(costs)
classif_mean = classif_errors.mean()
kwargs = dict(
dim=dim,
fappend=fappend,
df_min=df_min,
df_max=df_max,
confusion=confusion,
)
############################################################
# Extract method specific options.
############################################################
# TODO: SVMs only.
############################################################
# Load data.
data = load_data(dataset)
# Create object file names.
fname_args = []
if dim:
fname_args.append(str(dim))
fname_args.append(fappend)
mdl_fname = make_fname(METHOD, model, dataset, 'mdl', 'pk', *fname_args)
vec_fname = make_fname(METHOD, model, dataset, 'vec', 'pk', *fname_args)
dim_fname = make_fname(METHOD, model, dataset, 'dim', 'pk', *fname_args)
mdl_path = os.path.join(MODEL_HOME,mdl_fname)
vec_path = os.path.join(MODEL_HOME,vec_fname)
dim_path = os.path.join(MODEL_HOME,dim_fname)
model_files_present = os.path.isfile(mdl_path) and os.path.isfile(vec_path)
if dim:
results_path = output_folder + '/libsvm_results/'
if not os.path.exists(results_path):
os.makedirs(results_path)
#Factors = [1,1.5,2,2.5,3,3.5,4,4.5,5,6,7,8]
#Factors = [9,10,11,12,13,14,15,16,17,18,19,20]
Factors = [2000,1500,1000,900,800,700,600,500,400,300,200,100,75,50,40,30,20,10]
# measure the sensitivity of gamma for the selected brains and save the text file
brain_names = brain_list.keys()
results_file_c = 'libsvm_measures_factor.txt'
results_file_metadata = 'libsvm_measures_factor_metadata.txt'
for brain in brain_names[0:2]:
dataset_dir = os.path.join(os.environ.get('MLPYTHON_DATASET_REPO') + '/' + dataset_directory, brain)
all_data = data_utils.load_data(dataset_dir)
test = all_data['test']
fulltrain_backup = all_data['finaltrain']
resultc1, resultc2 = '' ,''
brain_str = brain + ' \n'
for factor in Factors:
resultc3 = ''
dice_t = np.zeros((10))
processed_timet = np.zeros((10))
for nb in range(10):
resultc3 = ''
all_data = data_utils.data_reduction(fulltrain_backup , factor)
all_data['test'] = test
print len(fulltrain_backup[0])
print len(all_data['finaltrain'][0])
def load_data(use_buckets=True):
print(FLAGS.data_dir)
if use_buckets:
return data_utils.load_data(FLAGS, _buckets)
else:
return data_utils.load_data(FLAGS, None)
def train(**kwargs):
"""
Train model
Load the whole train data in memory for faster operations
args: **kwargs (dict) keyword arguments that specify the model hyperparameters
"""
# Roll out the parameters
batch_size = kwargs["batch_size"]
n_batch_per_epoch = kwargs["n_batch_per_epoch"]
nb_epoch = kwargs["nb_epoch"]
model_name = kwargs["model_name"]
generator = kwargs["generator"]
image_dim_ordering = kwargs["image_dim_ordering"]
img_dim = kwargs["img_dim"]
patch_size = kwargs["patch_size"]
bn_mode = kwargs["bn_mode"]
label_smoothing = kwargs["use_label_smoothing"]
label_flipping = kwargs["label_flipping"]
dset = kwargs["dset"]
use_mbd = kwargs["use_mbd"]
epoch_size = n_batch_per_epoch * batch_size
# Setup environment (logging directory etc)
general_utils.setup_logging(model_name)
# Load and rescale data
X_full_train, X_sketch_train, X_full_val, X_sketch_val = data_utils.load_data(dset, image_dim_ordering)
img_dim = X_full_train.shape[-3:]
# Get the number of non overlapping patch and the size of input image to the discriminator
nb_patch, img_dim_disc = data_utils.get_nb_patch(img_dim, patch_size, image_dim_ordering)
try:
# Create optimizers
opt_dcgan = Adam(lr=1E-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
# opt_discriminator = SGD(lr=1E-3, momentum=0.9, nesterov=True)
opt_discriminator = Adam(lr=1E-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
# Load generator model
generator_model = models.load("generator_unet_%s" % generator,
img_dim,
nb_patch,
bn_mode,
use_mbd,
batch_size)
# Load discriminator model
discriminator_model = models.load("DCGAN_discriminator",
img_dim_disc,
nb_patch,
bn_mode,
use_mbd,
batch_size)
generator_model.compile(loss='mae', optimizer=opt_discriminator)
discriminator_model.trainable = False
DCGAN_model = models.DCGAN(generator_model,
discriminator_model,
img_dim,
patch_size,
image_dim_ordering)
loss = [l1_loss, 'binary_crossentropy']
loss_weights = [1E1, 1]
DCGAN_model.compile(loss=loss, loss_weights=loss_weights, optimizer=opt_dcgan)
discriminator_model.trainable = True
discriminator_model.compile(loss='binary_crossentropy', optimizer=opt_discriminator)
gen_loss = 100
disc_loss = 100
# Start training
print("Start training")
for e in range(nb_epoch):
# Initialize progbar and batch counter
progbar = generic_utils.Progbar(epoch_size)
batch_counter = 1
start = time.time()
for X_full_batch, X_sketch_batch in data_utils.gen_batch(X_full_train, X_sketch_train, batch_size):
# Create a batch to feed the discriminator model
X_disc, y_disc = data_utils.get_disc_batch(X_full_batch,
X_sketch_batch,
generator_model,
batch_counter,
patch_size,
image_dim_ordering,
label_smoothing=label_smoothing,
label_flipping=label_flipping)
# Update the discriminator
disc_loss = discriminator_model.train_on_batch(X_disc, y_disc)
# Create a batch to feed the generator model
X_gen_target, X_gen = next(data_utils.gen_batch(X_full_train, X_sketch_train, batch_size))
y_gen = np.zeros((X_gen.shape[0], 2), dtype=np.uint8)
y_gen[:, 1] = 1
# Freeze the discriminator
discriminator_model.trainable = False
gen_loss = DCGAN_model.train_on_batch(X_gen, [X_gen_target, y_gen])
# Unfreeze the discriminator
discriminator_model.trainable = True
batch_counter += 1
progbar.add(batch_size, values=[("D logloss", disc_loss),
("G tot", gen_loss[0]),
("G L1", gen_loss[1]),
("G logloss", gen_loss[2])])
# Save images for visualization
if batch_counter % (n_batch_per_epoch / 2) == 0:
# Get new images from validation
data_utils.plot_generated_batch(X_full_batch, X_sketch_batch, generator_model,
batch_size, image_dim_ordering, "training")
X_full_batch, X_sketch_batch = next(data_utils.gen_batch(X_full_val, X_sketch_val, batch_size))
data_utils.plot_generated_batch(X_full_batch, X_sketch_batch, generator_model,
batch_size, image_dim_ordering, "validation")
if batch_counter >= n_batch_per_epoch:
break
print("")
print('Epoch %s/%s, Time: %s' % (e + 1, nb_epoch, time.time() - start))
if e % 5 == 0:
gen_weights_path = os.path.join('../../models/%s/gen_weights_epoch%s.h5' % (model_name, e))
generator_model.save_weights(gen_weights_path, overwrite=True)
disc_weights_path = os.path.join('../../models/%s/disc_weights_epoch%s.h5' % (model_name, e))
discriminator_model.save_weights(disc_weights_path, overwrite=True)
DCGAN_weights_path = os.path.join('../../models/%s/DCGAN_weights_epoch%s.h5' % (model_name, e))
DCGAN_model.save_weights(DCGAN_weights_path, overwrite=True)
except KeyboardInterrupt:
pass
