def create_run_config(hp):
save_ckpt_steps = max(FLAGS.iterations_per_loop, FLAGS.local_eval_frequency)
save_ckpt_secs = FLAGS.save_checkpoints_secs or None
if save_ckpt_secs:
save_ckpt_steps = None
return trainer_lib.create_run_config(
model_dir=os.path.expanduser(FLAGS.output_dir),
master=FLAGS.master,
iterations_per_loop=FLAGS.iterations_per_loop,
num_shards=FLAGS.tpu_num_shards,
log_device_placement=FLAGS.log_device_placement,
save_checkpoints_steps=save_ckpt_steps,
save_checkpoints_secs=save_ckpt_secs,
keep_checkpoint_max=FLAGS.keep_checkpoint_max,
keep_checkpoint_every_n_hours=FLAGS.keep_checkpoint_every_n_hours,
num_gpus=FLAGS.worker_gpu,
gpu_order=FLAGS.gpu_order,
shard_to_cpu=FLAGS.locally_shard_to_cpu,
num_async_replicas=FLAGS.worker_replicas,
gpu_mem_fraction=FLAGS.worker_gpu_memory_fraction,
enable_graph_rewriter=FLAGS.enable_graph_rewriter,
use_tpu=FLAGS.use_tpu,
schedule=FLAGS.schedule,
no_data_parallelism=hp.no_data_parallelism,
daisy_chain_variables=hp.daisy_chain_variables,
ps_replicas=FLAGS.ps_replicas,
ps_job=FLAGS.ps_job,
ps_gpu=FLAGS.ps_gpu,
sync=FLAGS.sync,
worker_id=FLAGS.worker_id,
worker_job=FLAGS.worker_job,
random_seed=FLAGS.random_seed,
tpu_infeed_sleep_secs=FLAGS.tpu_infeed_sleep_secs)
def create_run_config(hp, params):
"""Create RunConfig"""
return trainer_lib.create_run_config(
master=params.master,
model_dir=params.model_dir,
iterations_per_loop=params.iterations_per_loop,
num_shards=params.tpu_num_shards,
log_device_placement=params.log_device_replacement,
save_checkpoints_steps=max(params.iterations_per_loop,
params.local_eval_frequency),
keep_checkpoint_max=params.keep_checkpoint_max,
keep_checkpoint_every_n_hours=params.keep_checkpoint_every_n_hours,
num_gpus=params.worker_gpu,
gpu_order=params.gpu_order,
shard_to_cpu=params.locally_shard_to_cpu,
num_async_replicas=params.worker_replicas,
enable_graph_rewriter=params.experimental_optimize_placement,
gpu_mem_fraction=params.worker_gpu_memory_fraction,
no_data_parallelism=params.no_data_parallelism,
daisy_chain_variables=params.daisy_chain_variables,
schedule=params.schedule,
worker_id=params.worker_id,
worker_job=params.worker_job,
ps_replicas=params.ps_replicas,
ps_job=params.ps_job,
ps_gpu=params.ps_gpu,
sync=params.sync,
use_tpu=params.use_tpu)
def testCompatibility(self):
model = "transformer"
hp_set = "transformer_test"
problem_name = "translate_ende_wmt8k"
hp = trainer_lib.create_hparams(
hp_set, data_dir=_DATA_DIR, problem_name=problem_name)
run_config = trainer_lib.create_run_config(model, model_dir=_CKPT_DIR)
estimator = trainer_lib.create_estimator(model, hp, run_config)
for prediction in estimator.predict(self.input_fn):
self.assertEqual(prediction["outputs"].dtype, np.int32)
def create_run_config(hp):
"""Create a run config.
Args:
hp: model hyperparameters
Returns:
a run config
"""
save_ckpt_steps = max(FLAGS.iterations_per_loop, FLAGS.local_eval_frequency)
save_ckpt_secs = FLAGS.save_checkpoints_secs or None
if save_ckpt_secs:
save_ckpt_steps = None
assert FLAGS.output_dir or FLAGS.checkpoint_path
tpu_config_extra_kwargs = {}
# the various custom getters we have written do not play well together yet.
# TODO(noam): ask rsepassi for help here.
daisy_chain_variables = (
hp.daisy_chain_variables and
hp.activation_dtype == "float32" and
hp.weight_dtype == "float32")
return trainer_lib.create_run_config(
model_dir=os.path.expanduser(FLAGS.output_dir),
master=FLAGS.master,
iterations_per_loop=FLAGS.iterations_per_loop,
num_shards=FLAGS.tpu_num_shards,
log_device_placement=FLAGS.log_device_placement,
save_checkpoints_steps=save_ckpt_steps,
save_checkpoints_secs=save_ckpt_secs,
keep_checkpoint_max=FLAGS.keep_checkpoint_max,
keep_checkpoint_every_n_hours=FLAGS.keep_checkpoint_every_n_hours,
num_gpus=FLAGS.worker_gpu,
gpu_order=FLAGS.gpu_order,
shard_to_cpu=FLAGS.locally_shard_to_cpu,
num_async_replicas=FLAGS.worker_replicas,
gpu_mem_fraction=FLAGS.worker_gpu_memory_fraction,
enable_graph_rewriter=FLAGS.enable_graph_rewriter,
use_tpu=FLAGS.use_tpu,
schedule=FLAGS.schedule,
no_data_parallelism=hp.no_data_parallelism,
daisy_chain_variables=daisy_chain_variables,
ps_replicas=FLAGS.ps_replicas,
ps_job=FLAGS.ps_job,
ps_gpu=FLAGS.ps_gpu,
sync=FLAGS.sync,
worker_id=FLAGS.worker_id,
worker_job=FLAGS.worker_job,
random_seed=FLAGS.random_seed,
tpu_infeed_sleep_secs=FLAGS.tpu_infeed_sleep_secs,
inter_op_parallelism_threads=FLAGS.inter_op_parallelism_threads,
log_step_count_steps=FLAGS.log_step_count_steps,
intra_op_parallelism_threads=FLAGS.intra_op_parallelism_threads,
tpu_config_extra_kwargs=tpu_config_extra_kwargs)
def create_surrogate_run_config(hp):
"""Create a run config.
Args:
hp: model hyperparameters
Returns:
a run config
"""
save_ckpt_steps = max(FLAGS.iterations_per_loop,
FLAGS.local_eval_frequency)
save_ckpt_secs = FLAGS.save_checkpoints_secs or None
if save_ckpt_secs:
save_ckpt_steps = None
assert FLAGS.surrogate_output_dir
# the various custom getters we have written do not play well together yet.
# TODO(noam): ask rsepassi for help here.
daisy_chain_variables = (hp.daisy_chain_variables
and hp.activation_dtype == "float32"
and hp.weight_dtype == "float32")
return trainer_lib.create_run_config(
model_dir=os.path.expanduser(FLAGS.surrogate_output_dir),
master=FLAGS.master,
iterations_per_loop=FLAGS.iterations_per_loop,
num_shards=FLAGS.tpu_num_shards,
log_device_placement=FLAGS.log_device_placement,
save_checkpoints_steps=save_ckpt_steps,
save_checkpoints_secs=save_ckpt_secs,
keep_checkpoint_max=FLAGS.keep_checkpoint_max,
keep_checkpoint_every_n_hours=FLAGS.keep_checkpoint_every_n_hours,
num_gpus=FLAGS.worker_gpu,
gpu_order=FLAGS.gpu_order,
shard_to_cpu=FLAGS.locally_shard_to_cpu,
num_async_replicas=FLAGS.worker_replicas,
gpu_mem_fraction=FLAGS.worker_gpu_memory_fraction,
enable_graph_rewriter=FLAGS.enable_graph_rewriter,
use_tpu=FLAGS.use_tpu,
schedule=FLAGS.schedule,
no_data_parallelism=hp.no_data_parallelism,
daisy_chain_variables=daisy_chain_variables,
ps_replicas=FLAGS.ps_replicas,
ps_job=FLAGS.ps_job,
ps_gpu=FLAGS.ps_gpu,
sync=FLAGS.sync,
worker_id=FLAGS.worker_id,
worker_job=FLAGS.worker_job,
random_seed=FLAGS.random_seed,
tpu_infeed_sleep_secs=FLAGS.tpu_infeed_sleep_secs,
inter_op_parallelism_threads=FLAGS.inter_op_parallelism_threads,
log_step_count_steps=FLAGS.log_step_count_steps,
intra_op_parallelism_threads=FLAGS.intra_op_parallelism_threads)
def train_supervised(problem, model_name, hparams, data_dir, output_dir,
train_steps, eval_steps, local_eval_frequency=None,
schedule="continuous_train_and_eval"):
"""Train supervised."""
if local_eval_frequency is None:
local_eval_frequency = getattr(FLAGS, "local_eval_frequency")
exp_fn = trainer_lib.create_experiment_fn(
model_name, problem, data_dir, train_steps, eval_steps,
min_eval_frequency=local_eval_frequency
)
run_config = trainer_lib.create_run_config(model_name, model_dir=output_dir)
exp = exp_fn(run_config, hparams)
getattr(exp, schedule)()
def testExperiment(self):
exp_fn = trainer_lib.create_experiment_fn(
"transformer",
"tiny_algo",
self.data_dir,
train_steps=1,
eval_steps=1,
min_eval_frequency=1,
use_tpu=False)
run_config = trainer_lib.create_run_config(
model_dir=self.data_dir, num_gpus=0, use_tpu=False)
hparams = registry.hparams("transformer_tiny_tpu")
exp = exp_fn(run_config, hparams)
exp.test()
def testExperiment(self):
exp_fn = trainer_lib.create_experiment_fn("transformer",
"tiny_algo",
self.data_dir,
train_steps=1,
eval_steps=1,
min_eval_frequency=1,
use_tpu=False)
run_config = trainer_lib.create_run_config(model_dir=self.data_dir,
num_gpus=0,
use_tpu=False)
hparams = registry.hparams("transformer_tiny_tpu")()
exp = exp_fn(run_config, hparams)
exp.test()
def train_supervised(problem, model_name, hparams, data_dir, output_dir,
train_steps, eval_steps, local_eval_frequency=None,
schedule="continuous_train_and_eval"):
"""Train supervised."""
if local_eval_frequency is None:
local_eval_frequency = FLAGS.local_eval_frequency
exp_fn = trainer_lib.create_experiment_fn(
model_name, problem, data_dir, train_steps, eval_steps,
min_eval_frequency=local_eval_frequency
)
run_config = trainer_lib.create_run_config(model_name, model_dir=output_dir)
exp = exp_fn(run_config, hparams)
getattr(exp, schedule)()
def testExperimentWithClass(self):
exp_fn = trainer_lib.create_experiment_fn(
"transformer",
algorithmic.TinyAlgo(),
algorithmic.TinyAlgo.data_dir,
train_steps=1,
eval_steps=1,
min_eval_frequency=1,
use_tpu=False)
run_config = trainer_lib.create_run_config(
model_name="transformer",
model_dir=algorithmic.TinyAlgo.data_dir,
num_gpus=0,
use_tpu=False)
hparams = registry.hparams("transformer_tiny_tpu")
exp = exp_fn(run_config, hparams)
exp.test()
def testExperimentWithClass(self):
exp_fn = trainer_lib.create_experiment_fn(
"transformer",
algorithmic.TinyAlgo(),
algorithmic.TinyAlgo.data_dir,
train_steps=1,
eval_steps=1,
min_eval_frequency=1,
use_tpu=False)
run_config = trainer_lib.create_run_config(
model_name="transformer",
model_dir=algorithmic.TinyAlgo.data_dir,
num_gpus=0,
use_tpu=False)
hparams = registry.hparams("transformer_tiny_tpu")
exp = exp_fn(run_config, hparams)
exp.test()
def _init_network(args):
with open(args.data_config) as fp:
config = json.load(fp)
CONFIG.update(config)
MODEL = "transformer"
HPARAMS = "transformer_big_tpu"
train_steps = 1000000
eval_steps = 10
save_checkpoints_steps = 10000
schedule = "continuous_train_and_eval"
hparams = create_hparams(HPARAMS)
print(json.loads(hparams.to_json()))
# fix TPU zone
from tensorflow.distribute.cluster_resolver import TPUClusterResolver
TPUClusterResolver.__init__.__defaults__ = (args.tpu_name, args.tpu_zone,
None, 'worker', None, None,
'default', None, None)
print(TPUClusterResolver.__init__.__defaults__)
RUN_CONFIG = create_run_config(
model_dir=args.model_dir,
model_name=MODEL,
save_checkpoints_steps=save_checkpoints_steps,
use_tpu=True,
cloud_tpu_name=args.tpu_name,
)
print(type(RUN_CONFIG))
tensorflow_exp_fn = create_experiment(
run_config=RUN_CONFIG,
hparams=hparams,
model_name=MODEL,
problem_name=TranslateManyToMany.name,
data_dir=args.data_dir,
train_steps=train_steps,
eval_steps=eval_steps,
use_tpu=True,
schedule=schedule,
#use_xla=True # For acceleration
)
return tensorflow_exp_fn
def main():
# print(registry.list_hparams())
data_dir = '../t2t_data/'
tmp_dir = '../data/'
TRAIN_DIR = '../logs_lm_new_t2t'
MODEL = 'transformer'
PROBLEM = 'pinyin2zh_problem'
tfe = tf.contrib.eager
tfe.enable_eager_execution()
pinyin2zh_problem = registry.problem(PROBLEM)
pinyin2zh_problem.generate_data(data_dir=data_dir, tmp_dir=tmp_dir)
hparams = trainer_lib.create_hparams("transformer_base")
hparams.batch_size = 4
hparams.learning_rate_warmup_steps = 45000
hparams.learning_rate = 0.0003
print(json.loads(hparams.to_json()))
# Initi Run COnfig for Model Training
RUN_CONFIG = create_run_config(
model_name=MODEL,
model_dir=TRAIN_DIR # Location of where model file is store
# More Params here in this fucntion for controling how noften to tave checkpoints and more.
)
# Create Tensorflow Experiment Object
tensorflow_exp_fn = create_experiment(
run_config=RUN_CONFIG,
hparams=hparams,
model_name=MODEL,
problem_name=PROBLEM,
data_dir=data_dir,
train_steps=400000, # Total number of train steps for all Epochs
eval_steps=100 # Number of steps to perform for each evaluation
)
# Kick off Training
tensorflow_exp_fn.train_and_evaluate()
def initialize(self, is_conditioned=False):
self.model_name = 'transformer'
self.hparams_set = 'transformer_tpu'
self.conditioned = is_conditioned
if self.conditioned:
self.ckpt_path = 'models/checkpoints/melody_conditioned_model_16.ckpt'
problem = MelodyToPianoPerformanceProblem()
else:
self.ckpt_path = 'models/checkpoints/unconditional_model_16.ckpt'
problem = PianoPerformanceLanguageModelProblem()
self.encoders = problem.get_feature_encoders()
# Set up hyperparams
hparams = trainer_lib.create_hparams(hparams_set=self.hparams_set)
trainer_lib.add_problem_hparams(hparams, problem)
hparams.num_hidden_layers = 16
hparams.sampling_method = 'random'
# Set up decoding hyperparams
decode_hparams = decoding.decode_hparams()
decode_hparams.alpha = 0.0
decode_hparams.beam_size = 1
if self.conditioned:
self.inputs = []
else:
self.targets = []
self.decode_length = 0
run_config = trainer_lib.create_run_config(hparams)
estimator = trainer_lib.create_estimator(
self.model_name, hparams, run_config,
decode_hparams=decode_hparams)
fnc = self.input_generation_conditional if self.conditioned else self.input_generator_unconditional
input_fn = decoding.make_input_fn_from_generator(fnc())
self.samples = estimator.predict(
input_fn, checkpoint_path=self.ckpt_path)
_ = next(self.samples)
def create_run_config(hp):
save_ckpt_steps = max(FLAGS.iterations_per_loop,
FLAGS.local_eval_frequency)
save_ckpt_secs = FLAGS.save_checkpoints_secs or None
if save_ckpt_secs:
save_ckpt_steps = None
assert FLAGS.output_dir or FLAGS.checkpoint_path
return trainer_lib.create_run_config(
model_dir=os.path.expanduser(FLAGS.output_dir),
master=FLAGS.master,
iterations_per_loop=FLAGS.iterations_per_loop,
num_shards=FLAGS.tpu_num_shards,
log_device_placement=FLAGS.log_device_placement,
save_checkpoints_steps=save_ckpt_steps,
save_checkpoints_secs=save_ckpt_secs,
keep_checkpoint_max=FLAGS.keep_checkpoint_max,
keep_checkpoint_every_n_hours=FLAGS.keep_checkpoint_every_n_hours,
num_gpus=FLAGS.worker_gpu,
gpu_order=FLAGS.gpu_order,
shard_to_cpu=FLAGS.locally_shard_to_cpu,
num_async_replicas=FLAGS.worker_replicas,
gpu_mem_fraction=FLAGS.worker_gpu_memory_fraction,
enable_graph_rewriter=FLAGS.enable_graph_rewriter,
use_tpu=FLAGS.use_tpu,
schedule=FLAGS.schedule,
no_data_parallelism=hp.no_data_parallelism,
daisy_chain_variables=hp.daisy_chain_variables,
ps_replicas=FLAGS.ps_replicas,
ps_job=FLAGS.ps_job,
ps_gpu=FLAGS.ps_gpu,
sync=FLAGS.sync,
worker_id=FLAGS.worker_id,
worker_job=FLAGS.worker_job,
random_seed=FLAGS.random_seed,
tpu_infeed_sleep_secs=FLAGS.tpu_infeed_sleep_secs,
inter_op_parallelism_threads=FLAGS.inter_op_parallelism_threads,
intra_op_parallelism_threads=FLAGS.intra_op_parallelism_threads)
def create_run_config(hp, output_dir=None):
"""Create a run config.
Args:
hp: model hyperparameters
output_dir: model's output directory, defaults to output_dir flag.
Returns:
a run config
"""
save_ckpt_steps = max(FLAGS.iterations_per_loop,
FLAGS.local_eval_frequency)
save_ckpt_secs = FLAGS.save_checkpoints_secs or None
if save_ckpt_secs:
save_ckpt_steps = None
assert FLAGS.output_dir or FLAGS.checkpoint_path
tpu_config_extra_kwargs = {}
if getattr(hp, "mtf_mode", False):
save_ckpt_steps = None # Disable the default saver
save_ckpt_secs = None # Disable the default saver
tpu_config_extra_kwargs = {
"num_cores_per_replica":
1,
"per_host_input_for_training":
tpu_config.InputPipelineConfig.BROADCAST,
}
# the various custom getters we have written do not play well together yet.
# TODO(noam): ask rsepassi for help here.
daisy_chain_variables = (hp.daisy_chain_variables
and hp.activation_dtype == "float32"
and hp.weight_dtype == "float32")
return trainer_lib.create_run_config(
model_name=FLAGS.model,
model_dir=output_dir or os.path.expanduser(FLAGS.output_dir),
master=FLAGS.master,
iterations_per_loop=FLAGS.iterations_per_loop,
num_shards=FLAGS.tpu_num_shards,
log_device_placement=FLAGS.log_device_placement,
save_checkpoints_steps=save_ckpt_steps,
save_checkpoints_secs=save_ckpt_secs,
keep_checkpoint_max=FLAGS.keep_checkpoint_max,
keep_checkpoint_every_n_hours=FLAGS.keep_checkpoint_every_n_hours,
num_gpus=FLAGS.worker_gpu,
gpu_order=FLAGS.gpu_order,
num_async_replicas=FLAGS.worker_replicas,
gpu_mem_fraction=FLAGS.worker_gpu_memory_fraction,
enable_graph_rewriter=FLAGS.enable_graph_rewriter,
use_tpu=FLAGS.use_tpu,
use_tpu_estimator=FLAGS.use_tpu_estimator,
xla_jit_level=FLAGS.xla_jit_level,
schedule=FLAGS.schedule,
no_data_parallelism=hp.no_data_parallelism,
optionally_use_dist_strat=FLAGS.optionally_use_dist_strat,
daisy_chain_variables=daisy_chain_variables,
ps_replicas=FLAGS.ps_replicas,
ps_job=FLAGS.ps_job,
ps_gpu=FLAGS.ps_gpu,
sync=FLAGS.sync,
worker_id=FLAGS.worker_id,
worker_job=FLAGS.worker_job,
random_seed=FLAGS.random_seed,
tpu_infeed_sleep_secs=FLAGS.tpu_infeed_sleep_secs,
inter_op_parallelism_threads=FLAGS.inter_op_parallelism_threads,
log_step_count_steps=FLAGS.log_step_count_steps,
intra_op_parallelism_threads=FLAGS.intra_op_parallelism_threads,
tpu_config_extra_kwargs=tpu_config_extra_kwargs,
cloud_tpu_name=FLAGS.cloud_tpu_name)
# tpu
hparams.symbol_modality_num_shards = 1
hparams.attention_dropout_broadcast_dims = '0,1'
hparams.relu_dropout_broadcast_dims = '1'
hparams.layer_prepostprocess_dropout_broadcast_dims = '1'
hparams.optimizer = 'Adafactor'
hparams.learning_rate_warmup_steps = 10000
hparams.learning_rate_schedule = 'rsqrt_decay'
hparams.warm_start_from_second = 'small-tatabahasa/model.ckpt'
print(hparams)
RUN_CONFIG = create_run_config(
model_dir=TRAIN_DIR,
model_name=MODEL,
save_checkpoints_steps=save_checkpoints_steps,
num_gpus=2,
)
tensorflow_exp_fn = create_experiment(
run_config=RUN_CONFIG,
hparams=hparams,
model_name=MODEL,
problem_name=PROBLEM,
data_dir=DATA_DIR,
train_steps=train_steps,
eval_steps=eval_steps,
# use_xla=True # For acceleration
)
tensorflow_exp_fn.train()
def music_generator(primer='erik_gnossienne',
primer_begin_buffer=10,
primer_length=90,
output_path='.',
filename='./public/output'):
SF2_PATH = './models/Yamaha-C5-Salamander-JNv5.1.sf2'
SAMPLE_RATE = 16000
# Upload a MIDI file and convert to NoteSequence.
def upload_midi():
data = list(files.upload().values())
if len(data) > 1:
print('Multiple files uploaded; using only one.')
return mm.midi_to_note_sequence(data[0])
# Decode a list of IDs.
def decode(ids, encoder):
ids = list(ids)
if text_encoder.EOS_ID in ids:
ids = ids[:ids.index(text_encoder.EOS_ID)]
return encoder.decode(ids)
model_name = 'transformer'
hparams_set = 'transformer_tpu'
ckpt_path = './models/checkpoints/unconditional_model_16.ckpt'
class PianoPerformanceLanguageModelProblem(score2perf.Score2PerfProblem):
@property
def add_eos_symbol(self):
return True
problem = PianoPerformanceLanguageModelProblem()
unconditional_encoders = problem.get_feature_encoders()
# Set up HParams.
hparams = trainer_lib.create_hparams(hparams_set=hparams_set)
trainer_lib.add_problem_hparams(hparams, problem)
hparams.num_hidden_layers = 16
hparams.sampling_method = 'random'
# Set up decoding HParams.
decode_hparams = decoding.decode_hparams()
decode_hparams.alpha = 0.0
decode_hparams.beam_size = 1
# Create Estimator.
run_config = trainer_lib.create_run_config(hparams)
estimator = trainer_lib.create_estimator(model_name,
hparams,
run_config,
decode_hparams=decode_hparams)
# These values will be changed by subsequent cells.
targets = []
decode_length = 0
# Create input generator (so we can adjust priming and
# decode length on the fly).
def input_generator():
global targets
global decode_length
while True:
yield {
'targets': np.array([targets], dtype=np.int32),
'decode_length': np.array(decode_length, dtype=np.int32)
}
# Start the Estimator, loading from the specified checkpoint.
input_fn = decoding.make_input_fn_from_generator(input_generator())
unconditional_samples = estimator.predict(input_fn,
checkpoint_path=ckpt_path)
# "Burn" one.
_ = next(unconditional_samples)
filenames = {
'C major arpeggio': './models/primers/c_major_arpeggio.mid',
'C major scale': './models/primers/c_major_scale.mid',
'Clair de Lune': './models/primers/clair_de_lune.mid',
'Classical':
'audio_midi/Classical_Piano_piano-midi.de_MIDIRip/bach/bach_846_format0.mid',
'erik_gymnopedie': 'audio_midi/erik_satie/gymnopedie_1_(c)oguri.mid',
'erik_gymnopedie_2': 'audio_midi/erik_satie/gymnopedie_2_(c)oguri.mid',
'erik_gymnopedie_3': 'audio_midi/erik_satie/gymnopedie_3_(c)oguri.mid',
'erik_gnossienne': 'audio_midi/erik_satie/gnossienne_1_(c)oguri.mid',
'erik_gnossienne_2': 'audio_midi/erik_satie/gnossienne_2_(c)oguri.mid',
'erik_gnossienne_3': 'audio_midi/erik_satie/gnossienne_3_(c)oguri.mid',
'erik_gnossienne_dery':
'audio_midi/erik_satie/gnossienne_1_(c)dery.mid',
'erik_gnossienne_dery_2':
'audio_midi/erik_satie/gnossienne_2_(c)dery.mid',
'erik_gnossienne_dery_3':
'audio_midi/erik_satie/gnossienne_3_(c)dery.mid',
'erik_gnossienne_dery_5':
'audio_midi/erik_satie/gnossienne_5_(c)dery.mid',
'erik_gnossienne_dery_6':
'audio_midi/erik_satie/gnossienne_6_(c)dery.mid',
'1': 'audio_midi/erik_satie/1.mid',
'2': 'audio_midi/erik_satie/2.mid',
'3': 'audio_midi/erik_satie/3.mid',
'4': 'audio_midi/erik_satie/4.mid',
'5': 'audio_midi/erik_satie/5.mid',
'6': 'audio_midi/erik_satie/6.mid',
'7': 'audio_midi/erik_satie/7.mid',
'8': 'audio_midi/erik_satie/8.mid',
'9': 'audio_midi/erik_satie/9.mid',
'10': 'audio_midi/erik_satie/10.mid',
}
# primer = 'C major scale'
#if primer == 'Upload your own!':
# primer_ns = upload_midi()
#else:
# # Use one of the provided primers.
# primer_ns = mm.midi_file_to_note_sequence(filenames[primer])
primer_ns = mm.midi_file_to_note_sequence(filenames[primer])
# Handle sustain pedal in the primer.
primer_ns = mm.apply_sustain_control_changes(primer_ns)
# Trim to desired number of seconds.
max_primer_seconds = primer_length
if primer_ns.total_time > max_primer_seconds:
print('Primer is longer than %d seconds, truncating.' %
max_primer_seconds)
primer_ns = mm.extract_subsequence(
primer_ns, primer_begin_buffer,
max_primer_seconds + primer_begin_buffer)
# Remove drums from primer if present.
if any(note.is_drum for note in primer_ns.notes):
print('Primer contains drums; they will be removed.')
notes = [note for note in primer_ns.notes if not note.is_drum]
del primer_ns.notes[:]
primer_ns.notes.extend(notes)
# Set primer instrument and program.
for note in primer_ns.notes:
note.instrument = 1
note.program = 0
## Play and plot the primer.
#mm.play_sequence(
# primer_ns,
# synth=mm.fluidsynth, sample_rate=SAMPLE_RATE, sf2_path=SF2_PATH)
#mm.plot_sequence(primer_ns)
mm.sequence_proto_to_midi_file(
primer_ns, join(output_path, 'primer_{}.mid'.format(filename)))
targets = unconditional_encoders['targets'].encode_note_sequence(primer_ns)
# Remove the end token from the encoded primer.
targets = targets[:-1]
decode_length = max(0, 10000 - len(targets))
if len(targets) >= 4096:
print(
'Primer has more events than maximum sequence length; nothing will be generated.'
)
# Generate sample events.
sample_ids = next(unconditional_samples)['outputs']
# Decode to NoteSequence.
midi_filename = decode(sample_ids,
encoder=unconditional_encoders['targets'])
ns = mm.midi_file_to_note_sequence(midi_filename)
print('Sample IDs: {}'.format(sample_ids))
print('Sample IDs length: {}'.format(len(sample_ids)))
print('Encoder: {}'.format(unconditional_encoders['targets']))
print('Unconditional Samples: {}'.format(unconditional_samples))
# print('{}'.format(ns))
# continuation_ns = mm.concatenate_sequences([primer_ns, ns])
continuation_ns = ns
# mm.play_sequence(
# continuation_ns,
# synth=mm.fluidsynth, sample_rate=SAMPLE_RATE, sf2_path=SF2_PATH)
# mm.plot_sequence(continuation_ns)
# try:
audio = mm.fluidsynth(continuation_ns,
sample_rate=SAMPLE_RATE,
sf2_path=SF2_PATH)
normalizer = float(np.iinfo(np.int16).max)
array_of_ints = np.array(np.asarray(audio) * normalizer, dtype=np.int16)
wavfile.write(join(output_path, filename + '.wav'), SAMPLE_RATE,
array_of_ints)
print('[+] Output stored as {}'.format(filename + '.wav'))
mm.sequence_proto_to_midi_file(
continuation_ns,
join(output_path, 'continuation_{}.mid'.format(filename)))
def run():
"""
Load Transformer model according to flags and start sampling.
:raises:
ValueError: if required flags are missing or invalid
"""
if FLAGS.model_path is None:
raise ValueError("Required Transformer pre-trained model path.")
if FLAGS.output_dir is None:
raise ValueError("Required MIDI output directory.")
if FLAGS.decode_length <= 0:
raise ValueError("Decode length must be > 0.")
problem = PianoPerformanceLanguageModelProblem()
unconditional_encoders = problem.get_feature_encoders()
primer_note_sequence = music_pb2.NoteSequence()
# It should be possible to supply absolutely no primer.
if FLAGS.primer_path is None:
targets = []
else:
primer_note_sequence = get_primer_ns(FLAGS.primer_path)
targets = unconditional_encoders["targets"].encode_note_sequence(
primer_note_sequence)
# Remove end token from encoded primer
targets = targets[:-1]
if len(targets) >= FLAGS.decode_length:
raise ValueError(
"Primer has more or equal events than max sequence length.")
decode_length = FLAGS.decode_length - len(targets)
# Set up hyperparameters
hparams = trainer_lib.create_hparams(
hparams_set="transformer_tpu") # Add flag
trainer_lib.add_problem_hparams(hparams, problem)
hparams.num_hidden_layers = NUM_HIDDEN_LAYERS
hparams.sampling_method = SAMPLING_METHOD
# Set up decoding HParams
decode_hparams = decoding.decode_hparams()
decode_hparams.alpha = ALPHA
decode_hparams.beam_size = BEAM_SIZE
# Create estimator
LOGGER.info("Loading model")
run_config = trainer_lib.create_run_config(hparams)
estimator = trainer_lib.create_estimator(
MODEL_NAME,
hparams,
run_config,
decode_hparams=decode_hparams,
)
generate(
estimator,
unconditional_encoders,
decode_length,
targets,
primer_note_sequence,
)
hparams.batch_size = 4048
hparams.learning_rate_warmup_steps = 40000
hparams.learning_rate = .2
save_checkpoints_steps = 10000
#keep_checkpoint_max = 100
# Can see all Hparams with code below
print(json.loads(hparams.to_json()))
# Init Run Config for Model Training
RUN_CONFIG = create_run_config(
model_dir=train_dir,
model_name=MODEL,
num_gpus=2,
#keep_checkpoint_max=keep_checkpoint_max,
save_checkpoints_steps=
save_checkpoints_steps # Location of where model file is store
# More Params here in this fucntion for controling how noften to tave checkpoints and more.
)
# # Create Tensorflow Experiment Object
tensorflow_exp_fn = create_experiment(
run_config=RUN_CONFIG,
hparams=hparams,
model_name=MODEL,
problem_name=PROBLEM,
data_dir=data_dir,
schedule="train_and_evaluate",
#eval_early_stopping_steps=5000,
min_eval_frequency=1000,
hparams.symbol_modality_num_shards = 1
hparams.attention_dropout_broadcast_dims = '0,1'
hparams.relu_dropout_broadcast_dims = '1'
hparams.layer_prepostprocess_dropout_broadcast_dims = '1'
hparams.optimizer = 'Adafactor'
hparams.learning_rate_warmup_steps = 10000
hparams.learning_rate_schedule = 'rsqrt_decay'
print(hparams)
RUN_CONFIG = create_run_config(
model_dir=TRAIN_DIR,
model_name=MODEL,
save_checkpoints_steps=save_checkpoints_steps,
use_tpu=True,
cloud_tpu_name='node-2',
iterations_per_loop=100,
schedule='train',
)
tensorflow_exp_fn = create_experiment(
run_config=RUN_CONFIG,
hparams=hparams,
model_name=MODEL,
problem_name=PROBLEM,
data_dir=DATA_DIR,
train_steps=train_steps,
eval_steps=eval_steps,
use_tpu=True,
use_tpu_estimator=False,
return text_problems.VocabType.CHARACTER
@property
def dataset_splits(self):
"""Splits of data to produce and number of output shards for each."""
# 10% evaluation data
return [{
"split": problem.DatasetSplit.TRAIN,
"shards": 9,
}, {
"split": problem.DatasetSplit.EVAL,
"shards": 1,
}]
run_config = trainer_lib.create_run_config()
hparams = trainer_lib.create_hparams(hparams_set="transformer_tiny",
data_dir=data_dir,
problem_name="poetry_chars")
estimator = trainer_lib.create_estimator('transformer', hparams, run_config)
def char_ids_gen(poem_config):
def gen():
char_gen = m.char_gen(poem_config)()
char_list = m.get_char_list(poem_config)
while True:
char = next(char_gen)
ind = None
from tensor2tensor.utils.trainer_lib import create_run_config, create_experiment
from problem import *
PROBLEM = 'translate_cmsc828'
MODEL = 'transformer'
HPARAMS = 'transformer_base'
TRAIN_DIR = './translator/model_files'
DATA_DIR = './translator/'
# Initi Run COnfig for Model Training
RUN_CONFIG = create_run_config(
model_dir=TRAIN_DIR, # Location of where model file is stored
model_name=MODEL,
# More Params here in this fucntion for controling how often to save checkpoints and more.
)
# Init Hparams object from T2T Problem
hparams = create_hparams(HPARAMS)
hparams.batch_size = 1024
# Create Tensorflow Experiment Object
tensorflow_exp_fn = create_experiment(
run_config=RUN_CONFIG,
hparams=hparams,
model_name=MODEL,
problem_name=PROBLEM,
# melody_problem = MelodyToPianoPerformanceProblem()
# melody_encoders = melody_problem.get_feature_encoders()
# Set up HParams.
hparams = trainer_lib.create_hparams(hparams_set=hparams_set)
trainer_lib.add_problem_hparams(hparams, uncondi_problem)
hparams.num_hidden_layers = 16
hparams.sampling_method = 'random'
# Set up decoding HParams.
decode_hparams = decoding.decode_hparams()
decode_hparams.alpha = 0.0
decode_hparams.beam_size = 1
# Create Estimator.
run_config = trainer_lib.create_run_config(hparams)
estimator = trainer_lib.create_estimator(model_name,
hparams,
run_config,
decode_hparams=decode_hparams)
# Create input generator (so we can adjust priming and
# decode length on the fly).
def input_generator():
global targets
global decode_length
while True:
yield {
'targets': np.array([targets], dtype=np.int32),
'decode_length': np.array(decode_length, dtype=np.int32)
hparams = create_hparams(HPARAMS)
# Make Changes to Hparams
hparams.batch_size = batch_size
hparams.learning_rate = ALPHA
#hparams.max_length = 256
# Can see all Hparams with code below
#print(json.loads(hparams.to_json())
gpus = 1 #should be changed only for MTF but not for ParDNN. ParDNN implicitly devides the model depending only on the placement file.
if len(sys.argv) >= 5:
gpus = int(sys.argv[4])
RUN_CONFIG = create_run_config(model_dir=TRAIN_DIR,
model_name=MODEL,
save_checkpoints_steps=save_checkpoints_steps,
num_gpus=gpus,
gpu_mem_fraction=0.97)
tensorflow_exp_fn = create_experiment(
run_config=RUN_CONFIG,
hparams=hparams,
model_name=MODEL,
problem_name=PROBLEM,
data_dir=DATA_DIR,
train_steps=train_steps,
eval_steps=eval_steps,
#use_xla=True # For acceleration
)
tensorflow_exp_fn.train_and_evaluate()
FLAGS.problems = problem_name
FLAGS.model = model_name
FLAGS.schedule = "train_and_evaluate"
FLAGS.save_checkpoints_secs = 0
FLAGS.local_eval_frequency = 2000
FLAGS.gpu_memory_fraction = .99
FLAGS.worker_gpu = 1
FLAGS.ps_gpu = 2
FLAGS.log_device_placement = True
FLAGS.worker_replicas = 2
RUN_CONFIG = trainer_lib.create_run_config(
model_dir=train_dir,
model_name="test",
keep_checkpoint_max=3,
save_checkpoints_secs=0,
gpu_mem_fraction=FLAGS.gpu_memory_fraction
)
exp_fn = trainer_lib.create_experiment(
run_config=RUN_CONFIG,
hparams=hparams,
model_name=model_name,
problem_name=problem_name,
data_dir=(data_dir),
train_steps=1000000,
eval_steps=100
)
exp_fn.train_and_evaluate()