def main():
st.title(" Script Buddy *v2*")
st.write("""
Film Script Language Generation with GPT2
***
""")
model, tokenizer = loader()
max_length = st.sidebar.slider(
""" Max Script Length
(Longer length, slower generation)""", 50, 1000)
context = st.sidebar.text_area("Context")
if st.sidebar.button("Generate"):
start_time = time.time()
if context:
sample = generate(model,
tokenizer,
input_text=context,
max_length=max_length)
else:
sample = generate(model, tokenizer, max_length=max_length)
end_time = time.time()
print(end_time - start_time)
else:
sample = ['']
st.text(sample[0])
def main():
args = parser.parse_args()
src_dir = os.path.join(DATA_DIR, 'noised_tgt')
dst_dir = os.path.join(DATA_DIR, 'tmp')
# clear
for name in os.listdir(dst_dir):
if name.endswith('.npy'):
os.remove(os.path.join(dst_dir, name))
# model
snapshot = torch.load(args.file, map_location=lambda s, _: s)
model = Generator(snapshot['channels'])
model.load_state_dict(snapshot['model'])
if args.gpu is not None:
model.cuda(args.gpu)
# generate
for name in sorted(os.listdir(os.path.join(src_dir))):
if not name.endswith('.npy'):
continue
src = os.path.join(src_dir, name)
dst = os.path.join(dst_dir, name[7:])
generate(model, src, dst, args.gpu)
# archive
with zipfile.ZipFile('submission.zip', 'w') as zip_writer:
for name in sorted(os.listdir(dst_dir)):
zip_writer.write(os.path.join(dst_dir, name), name)
def build(self):
num_fast_nodes = 0
for i in range(self.n_nodes):
if self.num_fast_nodes < num_fast_nodes:
self.nodes.append(Node(self.queue_size, 2))
num_fast_nodes +=1
else:
self.nodes.append(Node(self.queue_size, 1))
if self.verbose:
print("Generating Arrival and Service Times...")
for i in tqdm(range(self.num_arrivals)):
self.arrival_times.append(utils.generate(self.arrival_distn))
self.service_times.append(utils.generate(self.service_distn))
self.arrival_times.sort()
def train():
model_version, model_name = get_latest_model()
logger.info("Training on gathered game data, initializing from {}".format(
model_name))
new_model_name = generate(model_version + 1)
logger.info("New model will be {}".format(new_model_name))
save_file = os.path.join(PATHS.MODELS_DIR, new_model_name)
try:
logger.info("Getting tf_records")
tf_records = sorted(
gfile.Glob(os.path.join(PATHS.TRAINING_CHUNK_DIR,
'*.tfrecord.zz')))
tf_records = tf_records[-1 *
(GLOBAL_PARAMETER_STORE.WINDOW_SIZE //
GLOBAL_PARAMETER_STORE.EXAMPLES_PER_RECORD):]
print("Training from:", tf_records[0], "to", tf_records[-1])
with timer("Training"):
network.train(PATHS.ESTIMATOR_WORKING_DIR, tf_records,
model_version + 1)
network.export_latest_checkpoint_model(PATHS.ESTIMATOR_WORKING_DIR,
save_file)
except:
logger.info("Got an error training")
logging.exception("Train error")
def loop_outer(self):
steps = self.hp.steps_outer
interval = steps // 10
loader = self.loaders[Splits.train]
tracker = MetricsTracker(prefix=Splits.train)
stop_saver = EarlyStopSaver(self.net)
for i, batch in enumerate(
generate(loader, limit=steps, show_progress=True)):
self.opt_outer.zero_grad()
for task in MetaBatch(batch, self.device).get_tasks():
metrics = self.loop_inner(task)
tracker.store(metrics)
self.opt_outer.store_grad()
self.opt_outer.step(i)
if i % interval == 0:
metrics = tracker.get_average()
tracker.reset()
metrics.update(self.loop_eval(Splits.val))
print({k: round(v, 3) for k, v in metrics.items()})
stop_saver.check_stop(metrics["val_loss"])
self.save()
stop_saver.load_best()
self.save()
def home():
if request.method == "POST":
line = request.form["line"]
meme = request.form["meme"]
final = utils.generate(line,int(meme))
return render_template("response.html", image = final)
else:
return render_template("home.html")
def make_association(sender, instance, **kwargs):
if not instance.server_url or not instance.assoc_type:
raise Exception('association.server_url or association.assoc_type is None')
response = utils.generate(instance.server_url, assoc_type=instance.assoc_type)
instance.handle = response['handle']
instance.secret = response['secret']
instance.lifetime = response['lifetime']
instance.issued = int(time.time())
async def stream_files(request):
file = request.path_params.get("file")
return StreamingResponse(
generate(file),
headers={
"content-disposition": f"attachment; filename={file}",
"content-type": "application/octet-stream",
},
)
def parallel_generate(i):
data = []
labels = []
for j in range(1, RANGE):
grids = generate(ligands[j][0], ligands[j][1], proteins[i][0],
proteins[i][1], RADIUS, DISTANCE_THRESHOLD)
data.extend(grids)
label = 1 if i == j else 0
labels.extend([label] * (len(grids)))
return data, labels
def initialize_random_model():
bootstrap_name = generate(0)
bootstrap_model_path = os.path.join(PATHS.MODELS_DIR, bootstrap_name)
logger.info(
"Bootstrapping with working dir {}\n Model 0 exported to {}".format(
PATHS.ESTIMATOR_WORKING_DIR, bootstrap_model_path))
os.makedirs(PATHS.ESTIMATOR_WORKING_DIR, exist_ok=True)
os.makedirs(os.path.dirname(bootstrap_model_path), exist_ok=True)
network.initialize_random_model(PATHS.ESTIMATOR_WORKING_DIR)
network.export_latest_checkpoint_model(PATHS.ESTIMATOR_WORKING_DIR,
bootstrap_model_path)
def loop_inner(
self,
task: List[torch.Tensor],
bs: int,
steps: int,
) -> Dict[str, float]:
x_train, y_train, x_test, y_test = task
ds = TensorDataset(x_train, y_train)
loader = DataLoader(ds, bs, shuffle=True)
for batch in generate(loader, limit=steps):
self.run_batch(batch, is_train=True)
return self.run_batch((x_test, y_test), is_train=False)
def loop_inner(self, task: MetaTask) -> Dict[str, float]:
ds = RepeatTensorDataset(*task.train, num_repeat=self.hp.bs_inner)
loader = DataLoader(ds, self.hp.bs_inner, shuffle=True)
steps_per_epoch = self.hp.num_shots * self.hp.num_ways // self.hp.bs_inner
stopper = EarlyStopCallback(self.net)
for i, batch in enumerate(generate(loader, limit=self.hp.steps_inner)):
self.run_batch(batch, is_train=True)
if i % steps_per_epoch == 0 and self.hp.early_stop:
metrics = self.run_batch(task.val, is_train=False)
if stopper.check_stop(metrics["loss"]):
stopper.load_best()
break
return self.run_batch(task.test, is_train=False)
def loop_inner(self, task: MetaTask) -> Dict[str, float]:
ds = RepeatTensorDataset(*task.train, num_repeat=self.hp.bs_inner)
loader = DataLoader(ds, self.hp.bs_inner, shuffle=True)
steps_per_epoch = self.hp.num_shots * self.hp.num_ways // self.hp.bs_inner
stopper = EarlyStopCallback(self.net)
proto = self.get_prototypes(*task.train)
# calculate mean support for each embedding
for batch in generate(loader, limit=steps):
self.run_batch(batch, prototypes, is_train=True)
# run the different training method
return self.run_batch((x_test, y_test), prototypes, is_train=False)
def initialize(self):
self.first_layer_wts = utils.generate(self.hidden_layer_size, self.input_size)
self.first_layer_bias = utils.generate(self.hidden_layer_size, 1)
self.second_layer_wts = utils.generate(self.hidden_layer_size, self.hidden_layer_size)
self.second_layer_bias = utils.generate(self.hidden_layer_size, 1)
self.result_wts = utils.generate(self.output_size, self.hidden_layer_size)
self.result_bias = utils.generate(self.output_size, 1)
def generate():
sz = int(str(request.args.get('sz')))
e = ""
if request.args.get('e'):
e = int(str(request.args.get('e')))
if sz:
if request.args.get('e'):
ret = utils.generate(sz, e)
else:
ret = utils.generate(sz)
else:
return "Invalid"
final_ret = {
'n': ret[0],
'e': hex(ret[1]),
'p': ret[3],
'q': ret[4],
'dp_1': ret[5],
'dq_1': ret[6],
'coef': ret[7],
'd': hex(ret[9].d)
}
return json.dumps(final_ret)
def main(_):
config = flags.FLAGS
if config.mode == "train":
assert config.dataset in ("mnist", "cifar10")
config.in_shape = (config.batch_size, 32, 32, 3)
config.block_list = [eval(x) for x in config.block_list]
config.stride_list = [eval(x) for x in config.stride_list]
config.channel_list = [eval(x) for x in config.channel_list]
train(config)
elif config.mode == "debug":
config.train_steps = 1
config.viz_steps = 1
config.block_list = [2, 2, 2]
config.channel_list = [3, 4, 5]
config.stride_list = [1, 1, 2]
config.in_shape = (config.batch_size, 28, 28, 1)
train(config, debug=True)
elif config.mode == "prepare":
download_dataset(config)
elif config.mode == "sn":
test_spectral_norm()
elif config.mode == "iresnet":
test_iresnet()
elif config.mode == "trace":
test_trace_approximation()
elif config.mode == "inverse":
test_block_inversion()
elif config.mode == "squeeze":
test_squeeze()
elif config.mode == "trace_sn":
test_trace_sn()
elif config.mode == "generate":
generate(config)
elif config.mode == "reconstruct":
reconstruct(config)
def main():
axiom = axiom_init()
x, y = 0, length
while True:
display.fill(BG)
angle = 0
position = (0, 0)
for character in axiom:
for event in pygame.event.get():
if event.type == pygame.QUIT:
pygame.quit()
quit()
position, angle, x, y = turtle(draw, character,
position[0], position[1], angle, x, y)
axiom = generate(axiom)
clock.tick(1)
def generate_training_data():
data = []
labels = []
proteins = [[None, None]]
ligands = [[None, None]]
for i in range(1, RANGE):
p_coordinates, p_atom_types = read_pdb(
"training_data/{0}_pro_cg.pdb".format('%04d' % i))
l_coordinates, l_atom_types = read_pdb(
"training_data/{0}_lig_cg.pdb".format('%04d' % i))
proteins.append([p_coordinates, p_atom_types])
ligands.append([l_coordinates, l_atom_types])
for i in range(1, RANGE):
for j in range(1, RANGE):
grids = generate(ligands[j][0], ligands[j][1], proteins[i][0],
proteins[i][1], RADIUS, DISTANCE_THRESHOLD)
data.extend(grids)
label = 1 if i == j else 0
labels.extend([label] * (len(grids)))
return data, labels
def loop_outer(self):
steps = self.hparams.steps_outer
interval = steps // 10
loader = self.loaders[Splits.train]
tracker = MetricsTracker(prefix=Splits.train)
for i, batch in enumerate(
generate(loader, limit=steps, show_progress=True)):
self.opt_outer.zero_grad()
for task in MetaBatch(batch, self.device).get_tasks():
metrics = self.loop_inner(task, self.hparams.bs_inner,
self.hparams.steps_inner)
tracker.store(metrics)
self.opt_outer.store_grad()
self.opt_outer.step(i)
if i % interval == 0:
metrics = tracker.get_average()
tracker.reset()
metrics.update(self.loop_val())
print({k: round(v, 3) for k, v in metrics.items()})
def submit():
sequence = request.args.get("sequence", "MQVWPIEGIKKFETLSYLPP")
simple_url = generate(sequence)
simple_result = requests.get(simple_url)
print(f" Simple STATUS CODE: {simple_result.status_code}")
status = simple_result.status_code
attempt = 0
while status == 404 and attempt <= 300:
sleep(1)
print(attempt)
print(simple_url)
print(status)
simple_result = requests.get(simple_url)
status = simple_result.status_code
attempt += 1
if status == 200:
primary, secondary = parse_html(simple_result.content.decode("utf-8"))
secondary_list = structure_to_list(secondary)
return jsonify(primary_structure=primary,
secondary_structure=secondary,
secondary_list=secondary_list)
return "Sequence too long or JPRED API is down, try again later."
def on_epoch_end(self, epoch, logs={}):
epoch += self.offset
model.save_weights('checkpoints/model_weights_%d.h5' % epoch, overwrite=True)
# val_loss = logs['val_loss']
# write some sample generated text
seed_text = '\n'
diversities = [0.2, 0.5, 0.7, 1.0]
generateds = [utils.generate(model, seed_text, diversity, batch_size)
for diversity in diversities]
print(generateds[1])
filename = 'checkpoints/epoch_%d_gen.txt' % epoch
with open(filename, 'w') as f:
f.write('----- Epoch number %d\n' % epoch)
#f.write('-- Validation loss: %0.6f\n' % val_loss)
for generated,diversity in zip(generateds, diversities):
f.write('----- diversity: %0.1f\n' % diversity)
f.write(generated + '\n\n\n\n')
def main():
args = parse_args()
args.verbose = True
# prime file validation
if args.prime_file and not os.path.exists(args.prime_file):
utils.log(
'Error: prime file {} does not exist. Exiting.'.format(
args.prime_file), True)
exit(1)
else:
if not os.path.isdir(args.data_dir):
utils.log(
'Error: data dir {} does not exist. Exiting.'.format(
args.prime_file), True)
exit(1)
midi_files = [args.prime_file] if args.prime_file else \
[os.path.join(args.data_dir, f) for f in os.listdir(args.data_dir) \
if '.mid' in f or '.midi' in f]
experiment_dir = get_experiment_dir(args.experiment_dir)
utils.log('Using {} as --experiment_dir'.format(experiment_dir),
args.verbose)
if not args.save_dir:
args.save_dir = os.path.join(experiment_dir, 'generated')
if not os.path.isdir(args.save_dir):
os.makedirs(args.save_dir)
utils.log('Created directory {}'.format(args.save_dir), args.verbose)
if not args.from_checkpoint:
model, epoch = train.get_model(args, experiment_dir=experiment_dir)
utils.log(
'Model loaded from {}'.format(
os.path.join(experiment_dir, 'model.json')), args.verbose)
else:
# Load from checkpoint
with open(os.path.join(experiment_dir, 'model.json'), 'r') as f:
model = utils.model_from_json(f.read())
epoch = int(args.from_checkpoint)
newest_checkpoint = os.path.join(
experiment_dir,
f"checkpoints/checkpoint-epoch_{args.from_checkpoint}.hdf5")
utils.load_checkpoint(model, newest_checkpoint)
utils.log('Model loaded from checkpoint {}'.format(newest_checkpoint),
args.verbose)
window_size = model.layers[0].get_input_shape_at(0)[1]
seed_generator = utils.get_data_generator(
midi_files,
window_size=window_size,
batch_size=32,
num_threads=1,
use_instrument=args.use_instrument,
ignore_empty=args.ignore_empty,
encode_section=args.encode_section,
max_files_in_ram=10)
# validate midi instrument name
try:
# try and parse the instrument name as an int
instrument_num = int(args.midi_instrument)
if not (instrument_num >= 0 and instrument_num <= 127):
utils.log('Error: {} is not a supported instrument. Number values must be ' \
'be 0-127. Exiting'.format(args.midi_instrument), True)
exit(1)
args.midi_instrument = pretty_midi.program_to_instrument_name(
instrument_num)
except ValueError as err:
# if the instrument name is a string
try:
# validate that it can be converted to a program number
_ = pretty_midi.instrument_name_to_program(args.midi_instrument)
except ValueError as er:
utils.log('Error: {} is not a valid General MIDI instrument. Exiting.' \
.format(args.midi_instrument), True)
exit(1)
if args.multi_instruments:
if not args.prime_file:
utils.log(
'Error: You need to specify a prime file when generating a multi instrument track. Exiting.',
True)
exit(1)
utils.log(f"Sampling from single seed file: {args.prime_file}",
args.verbose)
generated_midi = pretty_midi.PrettyMIDI(initial_tempo=80)
source_midi = utils.parse_midi(args.prime_file)
melody_instruments = source_midi.instruments
# melody_instruments = utils.filter_monophonic(source_midi.instruments, 1.0)
for instrument in melody_instruments:
instrument_group = utils.get_family_id_by_instrument_normalized(
instrument.program)
# Get source track seed
X, y = [], []
windows = utils._encode_sliding_windows(instrument, window_size)
for w in windows:
if np.min(w[0][:, 0]) == 1:
# Window only contains pauses.. ignore!
continue
X.append(w[0])
if len(X) <= 5:
continue
seed = X[random.randint(0, len(X) - 1)]
# Generate track for this instrument
generated = []
buf = np.copy(seed).tolist()
while len(generated) < args.file_length:
buf_expanded = [x for x in buf]
# Add instrument class to input
if args.use_instrument:
buf_expanded = [[instrument_group] + x
for x in buf_expanded]
# Add section encoding to input
if args.encode_section:
sections = [0] * 4
active_section = int(
(len(generated) / args.file_length) * 4)
sections[active_section] = 1
buf_expanded = [sections + x for x in buf_expanded]
# Get prediction
arr = np.expand_dims(np.asarray(buf_expanded), 0)
pred = model.predict(arr)
# prob distribution sampling
index = np.random.choice(range(0, seed.shape[1]), p=pred[0])
pred = np.zeros(seed.shape[1])
pred[index] = 1
generated.append(pred)
buf.pop(0)
buf.append(pred.tolist())
# Create instrument
instrument = utils._network_output_to_instrument(
generated, instrument.program)
# Add to target midi
generated_midi.instruments.append(instrument)
if len(generated_midi.instruments) == 0:
raise Exception(
f"Found no monophonic instruments in {args.prime_file}")
# Save midi
time = datetime.now().strftime("%Y%m%d%H%M%S")
sample_name = f"{args.save_dir}/sampled_{time}.mid"
print(f"Writing generated sample to {sample_name}")
generated_midi.write(sample_name)
else:
# generate 10 tracks using random seeds
utils.log('Loading seed files...', args.verbose)
X, y = next(seed_generator)
generated = utils.generate(model,
X,
window_size,
args.file_length,
args.num_files,
args.midi_instrument,
use_instrument=args.use_instrument,
encode_section=args.encode_section)
for i, midi in enumerate(generated):
file = os.path.join(
args.save_dir,
f"{i+1}_instrument{midi.instruments[0].program}.mid")
midi.write(file.format(i + 1))
utils.log('wrote midi file to {}'.format(file), True)
# iteration
# curl from -L to log_2(2n)
# 2^L > n
# L > log_2(n)
# curl increase by one
# Maximum 'O(L)' iteration
if curl==diadic[0]:
# If there is an element for diadic expansion
coin = coin_list_packed.pop(0)
print("Select {}".format(str(coin)))
coin_ans.append(coin)
diadic.pop(0)
# O(1)
if curl<0:
coin_list_cur = generate(freq_list, curl)
# generate 'n' elements : O(n)
coin_list_packed = merge(coin_list_packed, coin_list_cur)
# len(coin_list_packed) < 2n
# len(coin_list_cur) = n
# Time complexity of merge operation : O(n)
coin_list_packed = package(coin_list_packed)
# packing
# pack => O(1)
# maximum len(coin_list_packed)/2 < n => O(n)
else:
coin_list_packed = package(coin_list_packed)
# packing
# pack => O(1)
# maximum len(coin_list_packed)/2 < n => O(n)
curl+=1
from random import randint
from faker import Factory
from utils import generate
fake = Factory.create('pl_PL')
if __name__ == '__main__':
template = """
<samochod>
<marka>{marka}</marka>
<model>{model}</model>
<rocznik>{rocznik}</rocznik>
<num_kat>{num_kat}</num_kat>
<cena>{cena}</cena>
</samochod>
"""
def gen_fun():
return template.format(**{
'marka': fake.text()[:10],
'model': fake.text()[:10],
'rocznik': randint(1970, 2000),
'num_kat': randint(1000, 2000),
'cena': randint(10000, 50000)
})
generate('cennik', gen_fun, 'ceny.xml')
model.summary()
callbacks = [
ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=4,
verbose=1,
min_lr=1e-9),
ModelCheckpoint(filepath=f'./Models/{steps}_new_model.h5',
monitor='val_acc',
save_best_only=False,
mode='max',
save_weights_only=False), history
]
gen = generate(batchsize=batchsize, steps=steps, T=T, sigma=sigma)
model.fit_generator(generator=gen,
steps_per_epoch=50,
epochs=25,
verbose=1,
callbacks=callbacks,
validation_data=generate(batchsize=batchsize,
steps=steps,
T=T,
sigma=sigma),
validation_steps=25)
train_loss = history.history['loss']
val_loss = history.history['val_loss']
train_acc = history.history['acc']
val_acc = history.history['val_acc']
if p < k:
s = p + 1
elif p > k:
e = p - 1
else:
return list[k]
return list[k]
if __name__ == "__main__":
#fix import issues
import sys
from os import path
sys.path.append( path.dirname( path.dirname( path.abspath(__file__) ) ) )
#import array generator
from utils import generate
a = generate(10, 100)
print(a)
min_el = quick_select(a, 0) # find min
median = quick_select(a, len(a)//2) #find the median
max_el = quick_select(a, len(a)-1) #find max
print("The min is = " , min_el)
print("The median is = " , median)
print("The max is = " , max_el)
cov_prime = cov.reshape(int(N / 3), 3, int(N / 3), 3)
cov = cov_prime.reshape(N, N, order='F')
results['cov'] = cov
results['means'] = means
# SVD covs
for i in range(k):
u, s, vt = np.linalg.svd(covs[i])
covs[i] = (u[:, :5], s[:5], vt[:5, :])
# Plot most probable
j = np.argmax(cluster_probs)
X_samples = []
for _ in range(25):
X_samples.append(generate(means[j], covs[j]))
X_samples_25 = np.array(X_samples)
results['X_samples'] = X_samples_25
results['most_probable'] = j
results['log_probs'] = []
results['X_test'] = X_test
results['position_kls'] = []
results['position_sample'] = []
results['velocity_longitudinal_kls'] = []
results['velocity_longitudinal_test'] = []
results['velocity_longitudinal_sample'] = []
def main(argv):
parser = argparse.ArgumentParser(
description='WikiText-2 language modeling')
parser.add_argument('--batch-size',
type=int,
default=70,
metavar='N',
help='input batch size for training (default: 90)'),
parser.add_argument('--eval-batch-size',
type=int,
default=50,
metavar='N',
help='input batch size for training (default: 50)'),
parser.add_argument('--save-directory',
type=str,
default='output/wikitext-2',
help='output directory')
parser.add_argument('--model-save-directory',
type=str,
default='../models/',
help='output directory')
parser.add_argument('--epochs',
type=int,
default=5,
metavar='N',
help='number of epochs to train')
parser.add_argument('--base-seq-len',
type=int,
default=70,
metavar='N',
help='Batch length'),
parser.add_argument('--min-seq-len',
type=int,
default=50,
metavar='N',
help='minimum batch length'),
parser.add_argument('--seq-prob',
type=int,
default=0.95,
metavar='N',
help='prob of being divided by 2'),
parser.add_argument('--seq-std',
type=int,
default=6,
metavar='N',
help='squence length std'),
parser.add_argument('--hidden-dim',
type=int,
default=1150,
metavar='N',
help='Hidden dim')
parser.add_argument('--embedding-dim',
type=int,
default=400,
metavar='N',
help='Embedding dim')
parser.add_argument('--lr',
type=int,
default=1e-4,
metavar='N',
help='learning rate'),
parser.add_argument('--weight-decay',
type=int,
default=2e-6,
metavar='N',
help='learning rate'),
parser.add_argument('--tag',
type=str,
default='testing.pt',
metavar='N',
help='learning rate'),
parser.add_argument('--no-cuda',
action='store_true',
default=False,
help='disables CUDA training')
args = parser.parse_args(argv)
args.cuda = not args.no_cuda and torch.cuda.is_available()
#load dataset
train_data, val_data, vocabulary = (
np.load('../dataset/wiki.train.npy')[:2],
np.load('../dataset/wiki.valid.npy')[:1],
np.load('../dataset/vocab.npy'))
word_count = len(vocabulary)
model = models.LSTMModelV2(word_count=word_count,
embedding_dim=embedding_dim,
hidden_dim=hidden_dim)
loss_fn = models.CrossEntropyLoss3D()
checkpoint_path = os.path.join(args.model_save_directory, args.tag)
if not os.path.exists(checkpoint_path):
model = models.LSTMModelV2(word_count=word_count,
embedding_dim=embedding_dim,
hidden_dim=hidden_dim)
else:
print("Using pre-trained model")
print("*" * 90)
model = models.LSTMModelV2(word_count=word_count,
embedding_dim=embedding_dim,
hidden_dim=hidden_dim)
checkpoint_path = os.path.join(args.model_save_directory, args.tag)
model.load_state_dict(torch.load(checkpoint_path))
if args.cuda:
model = model.cuda()
loss_fn = loss_fn.cuda()
generated = utils.generate(model,
sequence_length=10,
batch_size=2,
stochastic=True,
args=args).data.cpu().numpy()
utils.print_generated(utils.to_text(preds=generated,
vocabulary=vocabulary))
print('Model: ', model)
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
logging = dict()
logging['loss'] = []
logging['train_acc'] = []
logging['val_acc'] = []
model.train()
for epoch in range(args.epochs):
epoch_time = time.time()
np.random.shuffle(train_data)
train_data_ = utils.batchify(
utils.to_tensor(np.concatenate(train_data)), args.batch_size)
val_data_ = utils.batchify(utils.to_tensor(np.concatenate(val_data)),
args.eval_batch_size)
train_data_loader = utils.custom_data_loader(train_data_, args)
val_data_loader = utils.custom_data_loader(val_data_,
args,
evaluation=True)
#number of words
train_size = train_data_.size(0) * train_data_.size(1)
val_size = val_data_.size(0) * val_data_.size(1)
n_batchs = len(train_data_)
n_batchs_val = len(val_data_)
correct = 0
epoch_loss = 0
batch_index = 0
seq_len = 0
counter = 0
hidden = model.init_hidden(args.batch_size)
while (batch_index < n_batchs - 1):
optimizer.zero_grad()
X, y, seq_len = next(train_data_loader)
hidden = utils.repackage_hidden(hidden)
model.zero_grad()
out, hidden = model(X, hidden)
loss = loss_fn(out, y)
pred = out.data.max(2)[1].int().view(1, -1)
predicted = pred.eq(y.data.view_as(pred).int())
correct += predicted.sum()
loss.backward()
#scale lr with respect the size of the seq_len
utils.adjust_learning_rate(optimizer, args, seq_len)
torch.nn.utils.clip_grad_norm(model.parameters(), 0.25)
optimizer.step()
utils.adjust_learning_rate(optimizer, args, args.base_seq_len)
epoch_loss += loss.data.sum()
batch_index += seq_len
counter += 1
break
train_acc = correct / train_size
train_loss = epoch_loss / counter
val_acc = validate(model, val_data_loader, n_batchs_val) / val_size
logging['loss'].append(train_loss)
logging['train_acc'].append(train_acc)
logging['val_acc'].append(val_acc)
utils.save_model(model, checkpoint_path)
print('=' * 83)
print(
'| epoch {:3d} | time: {:5.2f}s | valid acc {:5.2f} | train acc {:5.2f} |'
'train loss {:8.2f}'.format(epoch + 1, (time.time() - epoch_time),
val_acc, train_acc, train_loss))
break
if s == None:
s = len(str(max(l)))
if p == None:
p = s
i = s - p
if i >= s:
return l
bins = [[] for _ in range(10)]
for e in l:
bins[int(str(e).zfill(s)[i])] += [e]
return flatten([radix_sort(b, p - 1, s) for b in bins])
if __name__ == "__main__":
results = {}
print("Radix Sort.")
for n in [500, 1000, 5000, 10000, 100000, 500000, 1000000]:
ts = time()
items = generate(n)
radix_sort(items)
te = time()
elapsed = (te - ts) * 1000.0
results[n] = elapsed
print(" with {} elements took {}".format(n, elapsed))
print(results)
data, ix2word, word2ix = load_dataset(opt.dataset_file)
#init model
model = PoetryModel(len(word2ix), 128, 256, num_layers=opt.num_layers)
#training
if opt.train_or_load:
model, loss_list=train(model=model, data=data, ix2word=ix2word, word2ix=word2ix, \
lr=opt.lr, batch_size=opt.batch_size, num_epochs=opt.num_epochs, \
device=opt.device)
#save model
torch.save(model.state_dict(), opt.save_path + '\\model.pt')
#draw loss-curve
draw_fig(loss_list, opt.save_path)
else:
#load model
model.load_state_dict(torch.load(opt.save_path + '\\model.pt'))
#generate acrostic
acrostic=gen_acrostic(model=model, start_words=opt.start_words_1, \
ix2word=ix2word, word2ix=word2ix, \
max_gen_len=opt.max_gen_len, device=opt.device)
#generate poetry
Poetry=generate(model=model, start_words=opt.start_words_2, \
ix2word=ix2word, word2ix=word2ix, \
max_gen_len=opt.max_gen_len, device=opt.device)
print(Poetry)
print()
print(acrostic)
from time import time
from utils import (
generate,
)
def insertion_sort(array):
"""This solution uses one `for` and one `while`."""
for i in range(1, len(array)):
key = i
j = i - 1
while j >= 0 and key < array[j]:
array[j + 1] = array[j]
j -= 1
array[j + 1] = key
return array
if __name__ == "__main__":
results = {}
print("Insertion Sort.")
for n in [500, 1000, 5000, 10000, 100000, 500000, 1000000]:
ts = time()
insertion_sort(generate(n))
te = time()
elapsed = (te-ts)*1000.0
results[n] = elapsed
print(" with {} elements took {}".format(n, elapsed))
print(results)
def main():
args = parse_args()
args.verbose = True
# prime file validation
if args.prime_file and not os.path.exists(args.prime_file):
utils.log(
'Error: prime file {} does not exist. Exiting.'.format(
args.prime_file), True)
exit(1)
else:
if not os.path.isdir(args.data_dir):
utils.log(
'Error: data dir {} does not exist. Exiting.'.format(
args.prime_file), True)
exit(1)
midi_files = [ args.prime_file ] if args.prime_file else \
[ os.path.join(args.data_dir, f) for f in os.listdir(args.data_dir) \
if '.mid' in f or '.midi' in f ]
experiment_dir = get_experiment_dir(args.experiment_dir)
utils.log('Using {} as --experiment_dir'.format(experiment_dir),
args.verbose)
if not args.save_dir:
args.save_dir = os.path.join(experiment_dir, 'generated')
if not os.path.isdir(args.save_dir):
os.makedirs(args.save_dir)
utils.log('Created directory {}'.format(args.save_dir), args.verbose)
model, epoch = train.get_model(args, experiment_dir=experiment_dir)
utils.log(
'Model loaded from {}'.format(
os.path.join(experiment_dir, 'model.json')), args.verbose)
window_size = model.layers[0].get_input_shape_at(0)[1]
seed_generator = utils.get_data_generator(midi_files,
window_size=window_size,
batch_size=32,
num_threads=1,
max_files_in_ram=10)
# validate midi instrument name
try:
# try and parse the instrument name as an int
instrument_num = int(args.midi_instrument)
if not (instrument_num >= 0 and instrument_num <= 127):
utils.log('Error: {} is not a supported instrument. Number values must be ' \
'be 0-127. Exiting'.format(args.midi_instrument), True)
exit(1)
args.midi_instrument = pretty_midi.program_to_instrument_name(
instrument_num)
except ValueError as err:
# if the instrument name is a string
try:
# validate that it can be converted to a program number
_ = pretty_midi.instrument_name_to_program(args.midi_instrument)
except ValueError as er:
utils.log('Error: {} is not a valid General MIDI instrument. Exiting.'\
.format(args.midi_instrument), True)
exit(1)
# generate 10 tracks using random seeds
utils.log('Loading seed files...', args.verbose)
X, y = next(seed_generator)
generated = utils.generate(model, X, window_size, args.file_length,
args.num_files, args.midi_instrument)
for i, midi in enumerate(generated):
file = os.path.join(args.save_dir, '{}.mid'.format(i + 1))
midi.write(file.format(i + 1))
utils.log('wrote midi file to {}'.format(file), True)
from random import randint
from faker import Factory
from utils import generate
fake = Factory.create('pl_PL')
if __name__ == '__main__':
template = """
<appartement id="{id}">
<address>{address}</address>
<city>{city}</city>
<area>{area}</area>
<rooms>{rooms}</rooms>
<year>{year}</year>
<price>{price}</price>
</appartement>
"""
root_node = 'appartement'
generate(root_node, template, {
'id': 1,
'address': fake.address(),
'city': fake.city(),
'year': randint(1970, 2000),
'rooms': randint(2, 10),
'area': randint(40, 150),
'price': randint(100000, 1000000)
}, 'test.xml')
window = 50 # length of window
length = 100 # number of events
number = 10 # number of samples
instrument = 'Acoustic Grand Piano' # full list is here https://www.midi.org/specifications/item/gm-level-1-sound-set
print('enter window size')
window = int(input())
print('enter lenght of sample')
length = int(input())
print('enter number of samples')
number = int(input())
print('enter instrument (for example Acoustic Grand Piano)')
instrument = input()
model = load_model('v5.hdf5') # here should be path to model
X, y = next(seed_generator)
generated = utils.generate(model, X, window, length, number, instrument)
if not os.path.isdir('output'):
os.makedirs('output')
for i, midi in enumerate(generated):
file = os.path.join('output', '{}.mid'.format(i + 1))
midi.write(file.format(i + 1))
fs = FluidSynth(
'FluidR3Mono_GM.sf3') # here should be full path to Sound Font file
fs.midi_to_audio(file.format(i + 1),
os.path.join('output', '{}.wav'.format(i + 1)))
