def main(_):
console.suppress_logging()
FLAGS.train = True
FLAGS.overwrite = True
# Start
console.start('MNIST VANILLA VAE')
# Get model
model = models.vanilla('vanilla_00')
if FLAGS.train:
mnist = load_mnist('../../data/MNIST',
flatten=True,
validation_size=0,
one_hot=True)
model.train(training_set=mnist[pedia.training],
epoch=1000,
batch_size=128,
print_cycle=50,
snapshot_cycle=200)
else:
samples = model.generate(sample_num=16)
console.show_status('{} samples generated'.format(samples.shape[0]))
imtool.gan_grid_plot(samples, show=True)
# End
console.end()
def main(_):
console.suppress_logging()
FLAGS.train = TRAIN
FLAGS.overwrite = OVERWRITE
console.start('EXP OB 01')
# Define system
system = define_system()
# Generate data
training_set, validation_set, test_set = generate_data(system)
if len(SYS_LOCK_ORDERS) == 1:
homogeneous_check(system, SYS_LOCK_ORDERS[0], training_set.signls[0],
training_set.responses[0])
# Identification
# .. wiener
wiener = Wiener(degree=WN_DEGREE, memory_depth=WN_MEN_DEPTH)
if WIENER_ON: wiener.identify(training_set, validation_set)
# .. vn
homo_strs = NN_HOMO_STRS
vns = collections.OrderedDict()
for homo_str in homo_strs:
console.show_status('Volterra Net h**o-strength = {:.2f}'.format(homo_str))
vn = init_vn('vn_{:.2f}{}'.format(homo_str, POSTFIX), homo_str=homo_str)
vns[homo_str] = vn
if FLAGS.train:
vn.identify(training_set, validation_set,
batch_size=50, print_cycle=100, epoch=EPOCH)
# Verification
verify(vns, wiener, system, test_set)
# End
console.end()
def show_selected_note_content(self):
note = self.selected_note
if note is not None:
console.show_status('Logs of selected note in header:')
console.split()
print(note.content)
console.split()
def main(_):
console.start('mlp task')
# Configurations
th = NlsHub(as_global=True)
th.memory_depth = 6
th.num_blocks = 2
th.multiplier = 2
th.hidden_dim = th.memory_depth * th.multiplier
# th.actype1 = 'lrelu' # Default: relu
th.epoch = 10
th.batch_size = 32
th.learning_rate = 1e-4
th.validation_per_round = 5
th.print_cycle = 100
th.train = True
# th.smart_train = True
# th.max_bad_apples = 4
# th.lr_decay = 0.6
th.early_stop = True
th.idle_tol = 20
th.save_mode = SaveMode.NAIVE
# th.warm_up_thres = 1
# th.at_most_save_once_per_round = True
th.overwrite = True
th.export_note = True
th.summary = False
# th.monitor = True
th.save_model = True
th.allow_growth = False
th.gpu_memory_fraction = 0.40
description = '0'
th.mark = 'mlp-{}x({}x{})-{}'.format(th.num_blocks, th.memory_depth,
th.multiplier, description)
# Get model
model = nlsf_model_lib.mlp_00(th)
# Load data
train_set, val_set, test_set = load_data(th.data_dir,
depth=th.memory_depth)
assert isinstance(train_set, DataSet)
assert isinstance(val_set, DataSet)
assert isinstance(test_set, DataSet)
# Train or evaluate
if th.train:
model.nn.train(train_set, validation_set=val_set, trainer_hub=th)
else:
console.show_status('Evaluating ...')
model.evaluate(train_set, start_at=th.memory_depth)
model.evaluate(val_set, start_at=th.memory_depth)
model.evaluate(test_set, start_at=th.memory_depth, plot=True)
# End
console.end()
def run(self, times=1, save=False, mark=''):
if self._sys_runs is not None:
times = checker.check_positive_integer(self._sys_runs)
console.show_status('Run # set to {}'.format(times))
# Set the corresponding flags if save
if save:
self.common_parameters['save_model'] = True
# Show parameters
self._show_parameters()
# Begin iteration
counter = 0
for run_id in range(times):
history = []
for hyper_dict in self._hyper_parameter_dicts():
# Set counter here
counter += 1
# Grand self._add_script_suffix the highest priority
if self._add_script_suffix is not None:
save = self._add_script_suffix
if save:
self.common_parameters['script_suffix'] = '_{}{}'.format(
mark, counter)
params = self._get_all_configs(hyper_dict)
self._apply_constraints(params)
params_list = self._get_config_strings(params)
params_string = ' '.join(params_list)
if params_string in history: continue
history.append(params_string)
console.show_status('Loading task ...',
'[Run {}/{}]'.format(run_id + 1, times))
call([self._python_cmd, self.module_name] + params_list)
# call(self.command_head + params_list)
print()
def save_selected_note(self, file_name):
from tframe.utils.note import Note
assert isinstance(file_name, str)
note = self.selected_note
assert isinstance(note, Note)
note.save(file_name)
console.show_status('Note saved to `{}`'.format(file_name))
def _link(self, x, **kwargs):
y, pkg = linker.sparse_affine(x,
self.num_neurons,
self.heads,
self.use_bit_max,
self._logits_initializer,
self._coef_initializer,
self._use_bias,
self._bias_initializer,
return_package=True)
# Encourage softmax activation to be saturated
ds_penalty = self._kwargs.get('desaturate_penalty', 0.0)
if ds_penalty > 0:
a = pkg['activation']
a_bar = tf.subtract(1.0, a)
context.add_loss_tensor(ds_penalty *
tf.reduce_mean(tf.minimum(a, a_bar)))
console.show_status(
'Desaturate penalty added in {}'.format(
tf.get_variable_scope().name), '++')
# Export variables
if hub.export_sparse_weights:
scope = '/'.join(tf.get_variable_scope().name.split('/')[1:])
# context.variables_to_export[scope + '/weights'] = pkg['weights']
# context.variables_to_export[scope + '/coef'] = pkg['coef']
context.weights_list.append(pkg['weights'])
return y
def _on_key_press(self, event):
assert isinstance(event, tk.Event)
flag = False
if event.keysym == 'Escape':
self.form.quit()
elif event.keysym == 'j':
flag = self._move_cursor(1)
elif event.keysym == 'k':
flag = self._move_cursor(-1)
elif event.keysym == 'quoteleft':
console.show_status('Widgets sizes:')
for k in self.__dict__.keys():
item = self.__dict__[k]
if isinstance(item, tk.Widget) or k == 'form':
str = '[{}] {}: {}x{}'.format(item.__class__, k,
item.winfo_height(),
item.winfo_width())
console.supplement(str)
elif event.keysym == 'Tab':
console.show_status('Data:')
for k in self.dataset._data.keys():
console.supplement('{}: {}'.format(
k, self.dataset._data[k].shape))
elif event.keysym == 'space':
self._resize()
else:
# console.show_status(event.keysym)
pass
# If needed, refresh image viewer
if flag:
self.refresh()
def on_key_press(viewer, event):
# Sanity check
assert isinstance(event, tk.Event)
assert isinstance(viewer, centre.SummaryViewer)
key_symbol = getattr(event, 'keysym')
if viewer.in_debug_mode:
on_key_press_debug(viewer, key_symbol)
if key_symbol == 'quoteleft':
console.show_status('Active flags:', symbol='::')
for k, v in viewer.config_panel.active_config_dict.items():
console.supplement('{}: {}'.format(k, v), level=2)
elif key_symbol in ('h', 'k'):
viewer.header.move_cursor(-1)
elif key_symbol in ('l', 'j'):
viewer.header.move_cursor(1)
elif key_symbol == 'n':
viewer.criteria_panel.move_between_groups(1)
elif key_symbol == 'p':
viewer.criteria_panel.move_between_groups(-1)
elif key_symbol == 'space':
viewer.header.show_selected_note_content()
elif key_symbol == 'Return':
viewer.header.on_label_detail_click()
elif key_symbol == 's':
file_name = tk.filedialog.asksaveasfilename(filetypes=[('Note file',
'.note')],
initialfile='untitled',
defaultextension='.note')
if file_name is not None:
viewer.header.save_selected_note(file_name)
def main(_):
console.suppress_logging()
FLAGS.overwrite = True
FLAGS.train = True
# Start
console.start()
# Get or define model
model = models.vanilla('vanilla_nov9_02_h2_c', bn=False)
# model = models.dcgan('dcgan_c00')
# model = models.vanilla_h3_rs_nbn('vanilla_nov9_01_h3_nbn_opdef')
# return
# Train or test
if FLAGS.train:
mnist = load_mnist('../../data/MNIST',
flatten=True,
validation_size=0,
one_hot=True)
model.train(training_set=mnist[pedia.training],
epoch=1000,
batch_size=128,
print_cycle=20,
snapshot_cycle=150,
sample_num=25)
else:
samples = model.generate(sample_num=16)
console.show_status('{} samples generated'.format(samples.shape[0]))
imtool.gan_grid_plot(samples, show=True)
# End
console.end()
def main(_):
console.suppress_logging()
FLAGS.train = True
FLAGS.overwrite = False
# Start
console.start('CIFAR-10 DCGAN')
# Get model
model = models.dcgan('dcgan_00')
if FLAGS.train:
cifar10 = load_cifar10('../../data/CIFAR-10',
validation_size=0,
one_hot=True)
model.train(training_set=cifar10[pedia.training],
epoch=20000,
batch_size=128,
print_cycle=20,
snapshot_cycle=2000)
else:
samples = model.generate(sample_num=16)
console.show_status('{} samples generated'.format(samples.shape[0]))
imtool.gan_grid_plot(samples, show=True)
# End
console.end()
def _outer_loop(self):
hub = self.th
rnd = 0
for _ in range(hub.total_outer_loops):
rnd += 1
if hub.progress_bar:
console.section('{} {}'.format(hub.round_name, rnd))
hub.tic()
# Begin round (RNN states will be reset here) # TODO
# self.model.begin_round(th=self.th)
# Do inner loop
self._inner_loop(rnd)
# End of round
if hub.progress_bar:
console.show_status(
'End of {}. Elapsed time is {:.1f} secs'.format(
hub.round_name, hub.toc()))
# Maybe give a report on metric
if hub.validation_on:
self.model.end_round(rnd)
if self.metric.get_idle_rounds(rnd) > self.th.patience:
self.th.raise_stop_flag()
# Advanced strategy
self._advanced_strategy(rnd)
# Export monitor info
if tfr.monitor.activated: tfr.monitor.export()
# Maybe save model
if self._save_model_at_round_end: self._save_model()
# Early stop
if hub.stop and self.model.bust(rnd): break
return rnd
def evaluate_model(self, data, batch_size=None, dynamic=False, **kwargs):
"""The word `evaluate` in this method name is different from that in
`self.evaluate` method. Here only eval_metric will be evaluated and
the result will be printed on terminal."""
# Check metric
if not self.eval_metric.activated:
raise AssertionError('!! Metric not defined')
# Do dynamic evaluation if necessary
if dynamic:
from tframe.trainers.eval_tools.dynamic_eval import DynamicEvaluator as de
de.dynamic_evaluate(self, data, kwargs.get('val_set', None),
kwargs.get('delay', None))
return
# If hub.val_progress_bar is True, this message will be showed in
# model.evaluate method
if not hub.val_progress_bar:
console.show_status('Evaluating on {} ...'.format(data.name))
# use val_progress_bar option here temporarily
result = self.validate_model(
data, batch_size, allow_sum=False,
verbose=hub.val_progress_bar)[self.eval_metric]
console.supplement('{} = {}'.format(
self.eval_metric.symbol, hub.decimal_str(result,
hub.val_decimals)))
return result
def main(_):
console.start('RNN task')
# Configurations
th = NlsHub(as_global=True)
th.memory_depth = 10
th.num_blocks = 1
th.multiplier = 8
th.hidden_dim = th.memory_depth * th.multiplier
th.num_steps = 32
th.epoch = 100000
th.batch_size = 32
th.learning_rate = 1e-4
th.validation_per_round = 20
th.print_cycle = 0
# th.train = False
th.smart_train = True
th.max_bad_apples = 4
th.lr_decay = 0.6
th.early_stop = True
th.idle_tol = 20
th.save_mode = SaveMode.ON_RECORD
th.warm_up_thres = 1
th.at_most_save_once_per_round = True
th.overwrite = True # Default: False
th.export_note = True
th.summary = True
th.monitor_preact = False
th.save_model = True
th.allow_growth = False
th.gpu_memory_fraction = 0.4
description = '0'
th.mark = 'rnn-{}x({}x{})-{}steps-{}'.format(
th.num_blocks, th.memory_depth, th.multiplier, th.num_steps, description)
# Get model
model = model_lib.rnn0(th)
# Load data
train_set, val_set, test_set = load_wiener_hammerstein(
th.data_dir, depth=th.memory_depth, validation_size=2000)
assert isinstance(train_set, DataSet)
assert isinstance(val_set, DataSet)
assert isinstance(test_set, DataSet)
# Train or evaluate
if th.train:
model.nn.train(train_set, validation_set=val_set, trainer_hub=th)
else:
console.show_status('Evaluating ...')
model.evaluate(train_set, start_at=th.memory_depth)
model.evaluate(val_set, start_at=th.memory_depth)
model.evaluate(test_set, start_at=th.memory_depth)
# End
console.end()
def identify(self, train_set, val_set=None):
# Sanity check
if not isinstance(train_set, DataSet):
raise TypeError('!! train_set must be a DataSet')
if val_set is not None and not isinstance(val_set, DataSet):
raise TypeError('!! val_set must be a DataSet')
if train_set.intensity is None:
raise TypeError('!! Intensity must not be None')
# Begin iteration
console.section('[Wiener] Begin Identification')
truth_norm, val_input, val_output = None, None, None
if val_set is not None:
val_input = val_set.signls[0]
val_output = val_set.responses[0]
truth_norm = val_output.norm
for i in range(len(train_set.signls)):
input_ = train_set.signls[i]
output = train_set.responses[i]
self.cross_correlation(input_, output, train_set.intensity)
status = 'Round {} finished. '.format(i + 1)
if val_set is not None:
pred = self(val_input)
delta = pred - val_output
err = delta.norm / truth_norm * 100
status += 'Error Ratio = {:.3f} %'.format(err)
console.show_status(status)
time.sleep(0.2)
console.show_status('Identification done.')
def _end_training(self, rounds):
if self.th.progress_bar: console.clear_line()
# If this is a hp-tuning task, write record summary
if self.th.hp_tuning:
assert not self.th.summary
self.key_metric.write_record_summary()
# Flush summary
if self.th.summary or self.th.hp_tuning:
self.model.agent.summary_writer.flush()
# Take notes
if self.is_online:
self.model.agent.take_notes(
'End training after {} iterations'.format(self.counter))
else:
total_round = ('' if self.total_rounds is None else
' ({:.1f} total)'.format(self.total_rounds))
self.model.agent.take_notes(
'End training after {} rounds{}'.format(rounds, total_round))
# Evaluate
if self._evaluate is not None:
# Load the best model if necessary
if self.th.save_model:
flag, _, _ = self.model.agent.load()
assert flag
# Evaluate model
self._evaluate(self)
# Show RAS if necessary
if self.th.lives > 0:
ras_info = self.metrics_manager.RAR_string
console.show_status(ras_info)
self.model.agent.take_notes(ras_info)
def _update_model(self, data_batch):
loss_dict = self.model.update_model(data_batch=data_batch)
loss_slots = [s for s in loss_dict.keys() if s.name == 'Loss']
# assert len(loss_slots) > 0
assert len(loss_slots) == 1
loss_slot = loss_slots[0]
self.batch_loss_stat.record(loss_dict[loss_slot])
# Record grads if necessary
# <monitor_grad_step_03: fetch and record>
if self.th.monitor_weight_grads:
grads = loss_dict.pop(self.model.grads_slot)
context.monitor.record_grads(grads)
# Record other tensors
if self.model.general_tensor_slot.activated:
tensors = loss_dict.pop(self.model.general_tensor_slot)
context.monitor.record_tensors(tensors)
# Check NaN
if self.th.terminate_on_nan:
for val in loss_dict.values():
if np.isnan(val):
msg = 'Forced termination triggered due to NAN in loss_dict'
console.show_status(msg)
self.model.agent.take_notes(msg)
self.th.force_terminate = True
break
return loss_dict
def compete(self, agent, rounds, opponent, title='Competition'):
console.show_status('[{}]'.format(title))
assert isinstance(agent, FMDPAgent)
rate, reports = agent.compete([self, opponent], rounds)
for report in reports:
console.supplement(report)
return rate
def _show_configurations(self):
console.show_status('Configurations:')
self.model.agent.take_notes('Configurations:', date_time=False)
for config in self.th.config_strings:
console.supplement(config)
self.model.agent.take_notes('.. {}'.format(config),
date_time=False)
def decrease_buffer_size(self, indices, is_training):
assert self.is_root and isinstance(indices, (list, tuple))
def _decrease(state):
assert isinstance(state, np.ndarray) and len(state) > len(indices)
return state[np.array(indices)]
# def _decrease(state):
# if isinstance(state, np.ndarray):
# assert len(state) > len(indices)
# return state[np.array(indices)]
# elif isinstance(state, (list, tuple)):
# # tf.scan returns a list of states
# state = list(state)
# for i, s in enumerate(state): state[i] = _decrease(s)
# return tuple(state)
# else: raise TypeError('!! Unknown type of states: {}'.format(type(state)))
# if is_training:
# self._train_state_buffer = _decrease(self._train_state_buffer)
# else: self._eval_state_buffer = _decrease(self._eval_state_buffer)
if is_training:
self._train_state_buffer = self._apply_to_nested_array(
self._train_state_buffer, _decrease)
else:
self._eval_state_buffer = self._apply_to_nested_array(
self._eval_state_buffer, _decrease)
# Display info if necessary
if 'partial_reset' in hub.verbose_config:
prefix = 'train' if is_training else 'eval'
console.show_status(
'Batch size of {} state buffer decreased'.format(prefix),
'[RNet]')
def main(_):
console.suppress_logging()
FLAGS.train = True
FLAGS.overwrite = False
# Start
console.start("MNIST DCGAN DEMO")
# Get model
model = models.dcgan('dcgan_002')
# model = models.dcgan_h3_rs_nbn()
# Train or test
if FLAGS.train:
mnist = load_mnist('../../data/MNIST', flatten=False, validation_size=0,
one_hot=True)
model.train(training_set=mnist[pedia.training], epoch=10, batch_size=128,
print_cycle=20, snapshot_cycle=200, D_times=1, G_times=1)
else:
samples = model.generate(sample_num=16)
console.show_status('{} samples generated'.format(samples.shape[0]))
imtool.gan_grid_plot(samples, show=True)
# End
console.end()
def main(_):
# Configuration
# FLAGS.train = False
# FLAGS.smart_train = True
FLAGS.overwrite = True
FLAGS.summary = True
FLAGS.save_model = False
FLAGS.snapshot = False
MEMORY_DEPTH = 1
EPOCH = 2
# Start
console.start('rnn_task')
# Initiate model
model = rnn_models.vanilla_RNN('rnn00')
# Load data
train_set, val_set, test_set = load_wiener_hammerstein(
r'../data/wiener_hammerstein/whb.tfd', depth=MEMORY_DEPTH)
assert isinstance(train_set, DataSet)
assert isinstance(val_set, DataSet)
assert isinstance(test_set, DataSet)
# Train or evaluate
if FLAGS.train:
pass
else:
console.show_status('Evaluating ...')
# End
console.end()
def separate_verified_data(data_dir,
csv_path,
to_path,
verbose=True,
vname='audio_train_verified',
uvname='audio_train_unverified'):
# Check path
check_path(data_dir, create_path=False)
check_path(csv_path, create_path=False)
# Read raw csv file and split
raw_csv = pd.read_csv(csv_path)
verified_csv = raw_csv.loc[raw_csv[MANUALLY_VERIFIED] == 1]
unverified_csv = raw_csv.loc[raw_csv[MANUALLY_VERIFIED] == 0]
# Copy verified data
for df, dn in ((verified_csv, vname), (unverified_csv, uvname)):
check_path(to_path, dn, create_path=True)
console.show_status('Generating {} data ... '.format(dn))
num_files = len(df)
for i, file_name in enumerate(df[FNAME]):
from_file = os.path.join(data_dir, file_name)
to_file = os.path.join(to_path, dn, file_name)
sh.copyfile(from_file, to_file)
if verbose: console.print_progress(i, num_files)
file_name = to_path + dn + '.csv'
df.to_csv(file_name, index=False)
console.show_status('CSV file saved to {}'.format(file_name))
def load_balanced_data(data_dir, train_size=20, validation_size=2,
init_f=None, round_len_f=None):
# Load data from data_dir
data_set = GPATBigData.load(data_dir, csv_path=core.v_train_csv_path,
lb_sheet_path=core.label_sheet_path)
assert isinstance(data_set, GPATBigData)
assert data_set.with_labels
# Pop subsets from data_set
train_set = data_set.pop_subset(
train_size, length_prone='long', name='train_set')
val_set = data_set.pop_subset(
validation_size, length_prone='short', name='validation_set')
train_set.init_f = init_f
train_set.round_len_f = round_len_f
val_set.init_f = init_f
val_set.round_len_f = round_len_f
# Show status
console.show_status('Train set (size={})'.format(train_set.size))
console.supplement('min_len = {}'.format(train_set.min_length))
console.supplement('max_len = {}'.format(train_set.max_length))
console.show_status('Validation set (size={})'.format(val_set.size))
console.supplement('min_len = {}'.format(val_set.min_length))
console.supplement('max_len = {}'.format(val_set.max_length))
return train_set.merge_to_signal_set(), val_set
def activate():
# Load datasets
train_set, val_set, test_set = du.load_data(th.data_dir)
# Calculate class weights
if th.class_weights is None and th.loss_string == 'wce':
train_targets = train_set.stack.targets.flatten()
samples_per_class = [
sum(train_targets == c) for c in range(th.num_classes)
]
class_weights = min(samples_per_class) / np.array(samples_per_class)
th.class_weights = class_weights
console.show_status('Class weights set to {}'.format(th.class_weights),
'++')
# Set input shape according to th.max_level and th.volume_only
du.FI2010.set_input_shape()
# Build model
assert callable(th.model)
model = th.model(th)
assert isinstance(model, Classifier)
# Train or evaluate
if th.train:
model.train(train_set,
validation_set=val_set,
test_set=test_set,
trainer_hub=th,
evaluate=lambda t: du.FI2010.evaluate(t, test_set))
else:
du.FI2010.evaluate(model, test_set)
# End
model.shutdown()
console.end()
def _on_detail_btn_click(self):
if len(self.value_list) == 0:
console.show_status('No notes with these criteria found under the '
'corresponding configures', '::')
return
console.show_status('{}:'.format(self.name), '::')
for v in np.sort(self.value_list):
console.supplement('{}'.format(v), level=2)
def homogeneous_check(model, order, input_, output):
console.show_status('Checking homogeneous system')
alpha = 2
console.supplement('Alpha = {}'.format(alpha))
truth = (alpha ** order) * output
delta = model(alpha * input_) - truth
ratio = delta.norm / truth.norm * 100
console.supplement('Error Ratio = {:.2f} %'.format(ratio))
def _print_progress(self, epi, start_time, steps, **kwargs):
"""Use a awkward way to avoid IDE warning :("""
console.clear_line()
console.show_status(
'Episode {} [{} total] {} steps, Time elapsed = {:.2f} sec'.format(
epi, self.counter, steps,
time.time() - start_time))
console.print_progress(epi, kwargs.get('total'))
def export_notes(self, filename='notes'):
assert hub.export_note
file_path = '{}/{}.txt'.format(self.note_dir, filename)
writer = open(file_path, 'a')
writer.write('=' * 79 + '\n')
writer.write(self._note.content + '\n')
writer.close()
console.show_status('Notes exported to {}'.format(file_path))
def _inter_cut(self, content, prompt='>>', start_time=None):
# Show content
console.show_status(content, symbol=prompt)
# Print progress bar
if self.th.progress_bar and self.th.round_length is not None:
assert isinstance(self._training_set, TFRData)
progress = self.th.round_progress
assert progress is not None
console.print_progress(progress=progress, start_time=start_time)
