def test_essay_batcher_2():
char = True
# char = False
U.seed_random(1234)
keep_unk = False
emb_dim = 100
emb_path = '/home/david/data/embed/glove.6B.{}d.txt'
vocab_dir = '/home/david/data/ets1b/2016'
vocab_file = os.path.join(vocab_dir, 'vocab_n250.txt')
data_dir = '/home/david/data/ats/ets'
id = 55433; essay_file = os.path.join(data_dir, '{0}', 'text.txt').format(id)
# id = 63986; essay_file = os.path.join(data_dir, '{0}', '{0}.txt.clean.tok').format(id)
reader = GlobReader(essay_file, chunk_size=1000, regex=REGEX_NUM, shuf=True)
if char:
word_vocab, char_vocab, max_word_length = Vocab.load_vocab(vocab_file)
text_parser = TextParser(word_vocab=word_vocab, char_vocab=char_vocab, keep_unk=keep_unk)
else:
E, word_vocab = Vocab.load_word_embeddings_ORIG(emb_path, emb_dim, essay_file, min_freq=5)
#E, word_vocab = Vocab.load_word_embeddings(emb_file, essay_file, min_freq=2)
text_parser = TextParser(word_vocab=word_vocab, keep_unk=keep_unk)
fields = {0:'id', 1:'y', -1:text_parser}
field_parser = FieldParser(fields, reader=reader, seed=1234)
batcher = EssayBatcher(reader=field_parser, batch_size=32, trim_words=True)
for b in batcher.batch_stream(stop=True, split_sentences=True):
print('{}\t{}'.format(b.w.shape, b.y.shape))
def main(config, dataset_root, resume):
set_print_precision()
seed_random()
config = get_config(config)
train_loader = get_data_loader(config, dataset_root, is_train=True)
test_loader = get_data_loader(config, dataset_root, is_train=False)
# model
model = StarGAN(config, train_loader, test_loader)
if not resume:
model.train_starGAN(init_epoch=0)
else:
model.resume_train() # resume train after the last saved epoch model
def main(config, mode):
seed_random()
set_numpy_precision()
if mode == 'random_sample':
model_type = SinGAN
elif mode == 'paint2image':
model_type = Paint2Image
elif mode == 'editing':
model_type = Editing
elif mode == 'harmonization':
model_type = Harmonization
elif mode == 'SR':
model_type = SuperResolution
else:
print("Invalid Parameter")
raise ValueError
model = model_type(get_config(config))
model.test_samples(save=True)
def test_response_batcher():
char = True
char = False
U.seed_random(1234)
keep_unk = False
emb_dir = '/home/david/data/embed'; emb_file = os.path.join(emb_dir, 'glove.6B.100d.txt')
emb_path = '/home/david/data/embed/glove.6B.{}d.txt'
vocab_dir = '/home/david/data/ets1b/2016'
vocab_file = os.path.join(vocab_dir, 'vocab_n250.txt')
data_dir = '/home/david/data/ats/ets'
id = 56375;
essay_file = os.path.join(data_dir, '{0}/text.txt').format(id)
# essay_file = os.path.join(data_dir, '{0}/{0}.txt').format(id)
reader = GlobReader(essay_file, chunk_size=1000, regex=REGEX_NUM, shuf=False)
if char:
word_vocab, char_vocab, max_word_length = Vocab.load_vocab(vocab_file)
text_parser = TextParser(word_vocab=word_vocab, char_vocab=char_vocab, keep_unk=keep_unk)
else:
E, word_vocab = Vocab.load_word_embeddings_ORIG(emb_path, 100, essay_file, min_freq=5)
#E, word_vocab = Vocab.load_word_embeddings(emb_file, essay_file, min_freq=2)
text_parser = TextParser(word_vocab=word_vocab, keep_unk=keep_unk)
fields = {0:'id', 1:'y', -1:text_parser}
field_parser = FieldParser(fields, reader=reader, seed=1234)
tot_vol = 0
batcher = ResponseBatcher(reader=field_parser, batch_size=64, trim_words=True)
for b in batcher.batch_stream(stop=True,
split_sentences=True,
spad='post',
wpad='post',
):
vol = np.prod(b.x.shape); tot_vol+=vol
print('{}\t{}\t{}'.format(vol, b.x.shape, b.y.shape))
print('tot_vol:\t{}'.format(tot_vol))
def parse_config(config_file, parser):
#parser = options.get_parser()
argv=[]# override config file here
FLAGS = get_config(parser=parser, config_file=config_file, argv=argv)
FLAGS.chkpt_dir = U.make_abs(FLAGS.chkpt_dir)
if FLAGS.load_model:
if FLAGS.load_chkpt_dir:
FLAGS.load_chkpt_dir = U.make_abs(FLAGS.load_chkpt_dir)
else:
FLAGS.load_chkpt_dir = FLAGS.chkpt_dir
else:
if FLAGS.model=='HANModel':
FLAGS.epoch_unfreeze_word = 0
FLAGS.cwd = os.getcwd()
FLAGS.log_file = os.path.abspath(os.path.join(FLAGS.cwd, 'log.txt'))
FLAGS.rand_seed = U.seed_random(FLAGS.rand_seed)
if FLAGS.id_dir is None:
FLAGS.id_dir = FLAGS.data_dir
else:
FLAGS.id_dir = os.path.join(FLAGS.data_dir, FLAGS.id_dir).format(FLAGS.item_id)
if FLAGS.attn_size>0:
FLAGS.mean_pool = False
if FLAGS.attn_type<0:
FLAGS.attn_type=0
if FLAGS.embed_type=='word':
FLAGS.model_std = None
FLAGS.attn_std = None
#### test ids
test_ids, test_id_file = None, None
FLAGS.test_y, FLAGS.test_yint = None, None
if FLAGS.test_pat is None:
FLAGS.save_test = None
FLAGS.load_test = None
else:
trait = ''
if FLAGS.trait is not None:
trait = '_{}'.format(FLAGS.trait)
test_id_file = os.path.join(FLAGS.data_dir, FLAGS.test_pat).format(FLAGS.item_id, trait)
#########################
if FLAGS.load_test and U.check_file(test_id_file):
#################################
data = U.read_cols(test_id_file)
test_ids = data[:,0]
if test_ids.dtype.name.startswith('float'):
test_ids = test_ids.astype('int32')
test_ids = test_ids.astype('unicode')
if data.shape[1]>1:
FLAGS.test_yint = data[:,1].astype('int32')
FLAGS.test_y = data[:,2].astype('float32')
#########################
if FLAGS.save_test and test_ids is not None:
FLAGS.save_test = False
# FLAGS.test_ids = set(test_ids) if test_ids is not None else []
FLAGS.test_ids = test_ids if test_ids is not None else []
FLAGS.test_id_file = test_id_file
''' don't overwrite MLT test ids!!! '''
if 'test_ids' in FLAGS.test_id_file:
FLAGS.save_test = False
#### valid ids
valid_ids, valid_id_file = None, None
if FLAGS.valid_pat is None:
FLAGS.save_valid = None
FLAGS.load_valid = None
else:
trait = ''
if FLAGS.trait is not None:
trait = '_{}'.format(FLAGS.trait)
valid_id_file = os.path.join(FLAGS.data_dir, FLAGS.valid_pat).format(FLAGS.item_id, trait)
if FLAGS.load_valid:
valid_ids = get_ids(valid_id_file)
if FLAGS.save_valid and valid_ids is not None:
FLAGS.save_valid = False
#FLAGS.valid_ids = set(valid_ids) if valid_ids is not None else []
FLAGS.valid_ids = valid_ids if valid_ids is not None else []
FLAGS.valid_id_file = valid_id_file
#### train ids
train_ids, train_id_file =None, None
if FLAGS.train_pat:
trait = ''
if FLAGS.trait is not None:
trait = '_{}'.format(FLAGS.trait)
train_id_file = os.path.join(FLAGS.data_dir, FLAGS.train_pat).format(FLAGS.item_id, trait)
train_ids = get_ids(train_id_file, default=[])
#FLAGS.train_ids = set(train_ids) if train_ids is not None else []
FLAGS.train_ids = train_ids if train_ids is not None else []
FLAGS.train_id_file = train_id_file
###################################
FLAGS.embed = U.adict({'type':FLAGS.embed_type,
'char':FLAGS.embed_type=='char',
'word':FLAGS.embed_type=='word' })
FLAGS.word_embed_dir = os.path.join(FLAGS.embed_dir, 'word')
FLAGS.char_embed_dir = os.path.join(FLAGS.embed_dir, 'char')
feats = ['kernel_widths','kernel_features','rnn_cells','rnn_sizes','rnn_bis','attn_sizes','attn_depths','attn_temps','pads','learning_rates']
for feat in feats:
if feat in FLAGS and FLAGS[feat]:
FLAGS[feat] = eval(eval(FLAGS[feat]))
FLAGS.wpad = FLAGS.pads[0]
FLAGS.spad = None if len(FLAGS.pads)<2 else FLAGS.pads[1]
if FLAGS.attn_depths[0]>1 or (len(FLAGS.attn_depths)>1 and FLAGS.attn_depths[1]>1):
FLAGS.attn_vis=False
if FLAGS.attn_sizes[0]<1:
FLAGS.attn_vis=False
if FLAGS.embed.char:
FLAGS.attn_vis=False
###################################
return FLAGS
def run():
plotter = Plotter(show_avgs=True, plot_elites=False)
fig_1 = 1
fig_2 = 1
fig_3 = 1
fig_4 = 1
fig_5 = 1
extension = 1
use_fixed_seed = 1
enable_hof = 0
enable_virulence = 0
generations = 600
export_folder = "../report/plots/"
generator_a = Generator(100, 1)
generator_b = Generator(10, 10)
generator_c = Generator(50, 2)
mutator = Mutator(mutation_rate=0.005, bit_flip=False)
selector = FitnessProportionateSelection()
n = 25
if enable_hof:
hof = HOF(scorer=Scorer(sample_size=10), size=50)
else:
hof = None
# Wrap selector with virulence handling
if enable_virulence:
selector = VirulenceSelector(selector, 0.75, normalise=True)
if fig_1:
seed = seed_random(0, use_fixed_seed)
description = "Figure 1 : [Seed {}]".format(seed)
print(description)
pop_a = generator_a.population(n, 0)
pop_b = generator_a.population(n, 100)
executor = Coevolution(scorer=F0Scorer(), selector=selector)
executor.run(pop_a, pop_b, generations)
plotter.make_plot(
executor,
fig_name=description,
export_path=os.path.join(export_folder, "fig1.png"),
)
if fig_2:
seed = seed_random(1, use_fixed_seed)
description = "Figure 2 : [Seed {}]".format(seed)
print(description)
pop_a = generator_a.population(n, 0)
pop_b = generator_a.population(n, 0)
executor = Coevolution(mutator=mutator, hof=hof, selector=selector)
executor.run(pop_a, pop_b, generations)
plotter.make_plot(
executor,
fig_name=description,
export_path=os.path.join(export_folder, "fig2.png"),
)
if fig_3:
seed = seed_random(8486058433753192762, use_fixed_seed)
description = "Figure 3 : [Seed {}]".format(seed)
print(description)
pop_a = generator_a.population(n, 0)
pop_b = generator_a.population(n, 0)
executor = Coevolution(
mutator=mutator, scorer=Scorer(sample_size=1), hof=hof, selector=selector
)
executor.run(pop_a, pop_b, generations)
plotter.make_plot(
executor,
fig_name=description,
export_path=os.path.join(export_folder, "fig3.png"),
)
if fig_4:
seed = seed_random(59759543964706904, use_fixed_seed)
description = "Figure 4 : [Seed {}]".format(seed)
print(description)
pop_a = generator_b.population(n, 0)
pop_b = generator_b.population(n, 0)
executor = Coevolution(mutator=mutator, hof=hof, selector=selector)
executor.run(pop_a, pop_b, generations)
plotter.make_plot(
executor,
fig_name=description,
export_path=os.path.join(export_folder, "fig4.png"),
)
if fig_5:
seed = seed_random(5706501168717675099, use_fixed_seed)
description = "Figure 5 : [Seed {}]".format(seed)
print(description)
pop_a = generator_c.population(n, 0)
pop_b = generator_c.population(n, 0)
executor = Coevolution(
mutator=mutator,
scorer=Scorer(intransitive=True),
hof=hof,
selector=selector,
)
executor.run(pop_a, pop_b, generations)
plotter.make_plot(
executor,
fig_name=description,
export_path=os.path.join(export_folder, "fig5.png"),
)
if extension:
hof = HOF(scorer=Scorer(sample_size=10), size=50)
generations = 1200
fp_selector = FitnessProportionateSelection()
sus_selector = StochasticUniversalSampling()
t_selector = TournamentSelection(3)
ext_cfgs = []
ext_cfgs += [
(
"Virulence [0.5]",
"fig_5_v0.5.png",
VirulenceSelector(fp_selector, 0.5),
False,
)
]
ext_cfgs += [
(
"Virulence [0.75]",
"fig_5_v0.75.png",
VirulenceSelector(fp_selector, 0.75),
False,
)
]
ext_cfgs += [
(
"Virulence [0.75] + SUS",
"fig_5_v0.75_sus.png",
VirulenceSelector(sus_selector, 0.75),
False,
)
]
ext_cfgs += [
(
"Virulence [0.75] + TS",
"fig_5_v0.75_ts.png",
VirulenceSelector(t_selector, 0.75),
False,
)
]
ext_cfgs += [
(
"Virulence [0.75] + SUS + HOF",
"fig_5_v0.75_sus_hof.png",
VirulenceSelector(sus_selector, 0.75),
True,
)
]
ext_cfgs += [
(
"Virulence [0.75] + HOF",
"fig_5_v0.75_hof.png",
VirulenceSelector(fp_selector, 0.75),
True,
)
]
ext_cfgs += [("HOF", "fig_5_hof.png", fp_selector, True)]
for (cfg_txt, export_name, selector, use_hof) in ext_cfgs:
seed = seed_random(8985012493578745191, use_fixed_seed)
description = "Figure 5 - {} : [Seed {}]".format(cfg_txt, seed)
print(description)
pop_a = generator_c.population(n, 0)
pop_b = generator_c.population(n, 0)
executor = Coevolution(
mutator=mutator,
scorer=Scorer(intransitive=True),
selector=selector,
hof=hof if use_hof else None,
)
executor.run(pop_a, pop_b, generations)
plotter.make_plot(
executor,
fig_name=description,
export_path=os.path.join(export_folder, export_name),
)