def load_images(self, img_size):
if os.path.exists(self.data_path + f"/image_data_{img_size}.plk"):
with open(self.data_path + f"/image_data_{img_size}.plk",
"rb") as fp:
data = pickle.load(fp)
else:
file_name = "CUB_200_2011.tgz"
url = "http://www.vision.caltech.edu.s3-us-west-2.amazonaws.com/visipedia-data/CUB-200-2011/CUB_200_2011.tgz"
if not os.path.exists(self.data_path + f"/{file_name}"):
downloder(url, self.data_path + f"/{file_name}")
if len(glob.glob(self.data_path + "/images/*.jpg")) != IMAGE_SIZE:
print("Info:Extracting image data from tar file")
import tarfile, shutil
with tarfile.open(self.data_path + f"/{file_name}",
'r') as tar_fp:
tar_fp.extractall(self.data_path)
shutil.move(self.data_path + "/CUB_200_2011/images",
self.data_path)
data = {}
files = glob.glob(self.data_path + "/images/*/*.jpg")
print(f"Info:load {img_size}x{img_size} image data")
for i, _path in enumerate(files):
arr_id = int(_path.split("_")[-1].split(".")[0])
arr = self.path2array(_path, img_size)
data[arr_id] = arr
progress(i + 1, IMAGE_SIZE)
print("")
with open(self.data_path + f"/image_data_{img_size}.plk",
"wb") as fp:
pickle.dump(data, fp)
return data
def deep_train(data_iterator, model, epoch=0, steps=500):
'''
training for one epoch
'''
optimizer = optim.Adam(model.parameters(), lr=0.001)
print('Start training epoch %i.....' % epoch)
losses = []
for step in range(steps):
start = time.time()
optimizer.zero_grad()
# get data
X, y = next(data_iterator)
X, y = tensor_convertor(X, y=y)
assert X.shape==(data_iterator.batch_size,5, 400) or \
X.shape==(data_iterator.batch_size + 1,5,400), \
X.shape
#get prediction (forward)
pred_y = model(X)
pred_y = pred_y.view(-1)
assert sum(pred_y != pred_y).item() == 0, pred_y
loss = F.binary_cross_entropy(pred_y, y)
losses.append(loss)
# update gradient
loss.backward()
optimizer.step()
end = time.time()
status = '%i/%i step\tloss:%.3f\tused: %.3fs' \
%(step, steps, loss.item(), end-start)
progress(steps, step, epoch + 1, status)
calculate_metrics(y, pred_y, epoch + 1, loss.item())
return losses
def send_requests(db, known_valid, test_case_1, test_case_2, missing_chars):
"""
:param db: The database where samples are stored
:param known_valid: The known valid characters
:param test_case_1: One character to test (test case 1)
:param test_case_2: One character to test (test case 2)
:param missing_chars: The total number of chars of the API key
:return: None. All is stored in the DB
"""
session = requests.Session()
http_adapter = requests.adapters.HTTPAdapter(max_retries=3)
session.mount('http://', http_adapter)
session.mount('https://', http_adapter)
token_test_case_1 = generate_test_token(known_valid, test_case_1,
missing_chars)
token_test_case_2 = generate_test_token(known_valid, test_case_2,
missing_chars)
print('Collecting %s samples for:' % NUM_SAMPLES * 2)
print(' - %s' % token_test_case_1)
print(' - %s' % token_test_case_2)
print('')
print('Test name: %s' % TEST_NAME)
for i in xrange(NUM_SAMPLES):
#
# What I'm trying to do here is to get timings in pairs.
# https://github.com/andresriancho/django-rest-framework-timing/issues/5
#
tmp_results = {}
# Sending the HTTP requests in different order during sample capture is
# something recommended by Paul McMillan and Sebastian Schinzel, they
# recommend it because it might break some caches
shuffled_token_tests = [(0, token_test_case_1), (1, token_test_case_2)]
random.shuffle(shuffled_token_tests)
for j, token in shuffled_token_tests:
response, naive_time = send_with_naive_timing(session, URL, token)
tmp_results[j] = (response, naive_time, token)
data = {'test_name': TEST_NAME, 'capture_timestamp': time.time()}
for j, (response, naive_time,
token) in enumerate(tmp_results.values()):
data.update({
'x_runtime_%s' % j: response.headers['X-Runtime'],
'userspace_rtt_microseconds_%s' % j: naive_time,
'token_%s' % j: token
})
db.insert(data)
if i % (NUM_SAMPLES / 1000) == 0:
progress(i, NUM_SAMPLES)
def index_blog(self):
ghost = get_voyage_connection()
with ghost.cursor() as ghost_cur:
ghost_cur.execute("SELECT id, title, published_at, html "
"FROM posts WHERE status='published' "
"ORDER BY published_at DESC")
posts = [{
"post_id": str(post[0]),
"title": post[1],
"text": clean_string(post[3])
} for post in ghost_cur.fetchall()]
total_cnt = len(posts)
print("Index %d posts" % total_cnt)
for i, post in enumerate(posts):
ghost_cur.execute(
"SELECT t.name FROM tags t "
"LEFT JOIN posts_tags pt ON pt.tag_id=t.id "
"WHERE pt.post_id=%s", str(post['post_id']))
tags = [tag[0] for tag in ghost_cur.fetchall()]
post["tags"] = ",".join(tags)
suffix = post["title"][:25]
progress(i, total_cnt, suffix)
self.index_document(self.index_name, post)
ghost.close()
print("\nFinished indexing posts")
def load_images(self, is_train=True):
if is_train:
data_type = "train"
data_len = train_image_size
else:
data_type = "val"
data_len = val_image_size
path = self.data_path + "/image"
if not os.path.exists(path): os.mkdir(path)
if os.path.exists(path +
f"/{data_type}2014_array/{self.image_size[0]}"):
files = glob.glob(
path + f"/{data_type}2014_array/{self.image_size[0]}/*.npy")
id_list = [
int(path.split("_")[-1].split(".")[0]) for path in files
]
data = {id: path for id, path in zip(id_list, files)}
return data
else:
url = f"http://images.cocodataset.org/zips/{data_type}2014.zip"
file_name = url.split("/")[-1]
if not os.path.exists(path + f"/{file_name}"):
downloder(url, path + f"/{file_name}")
if len(glob.glob(path + f"/{data_type}2014/*.jpg")) != data_len:
print(f"Info:Extracting {data_type} image data from zip file")
import zipfile
with zipfile.ZipFile(path + f"/{file_name}") as zip_fp:
zip_fp.extractall(path)
files = glob.glob(path + f"/{data_type}2014/*.jpg")
path = path + f"/{data_type}2014_array"
if not os.path.exists(path): os.mkdir(path)
path = path + f"/{self.image_size[0]}"
if not os.path.exists(path):
os.mkdir(path)
print(f"Info:Conveting {data_type} image data path to ndarray")
for i, _path in enumerate(files):
file_name = _path.split("/")[-1].split(".")[0]
arr = self.path2array(_path)
np.save(path + f"/{file_name}.npy", arr)
progress(i + 1, data_len)
print("")
files = glob.glob(path + "/*.npy")
id_list = [
int(path.split("_")[-1].split(".")[0]) for path in files
]
data = {id: path for id, path in zip(id_list, files)}
return data
def load_images(self, is_train=True):
if is_train:
data_type = "train"
data_len = train_image_size
else:
data_type = "val"
data_len = val_image_size
if len(glob.glob(self.data_path +
f"/{data_type}2014_array/*.npy")) == data_len:
files = glob.glob(self.data_path + f"/{data_type}2014_array/*.npy")
id_list = [
int(path.split("_")[-1].split(".")[0]) for path in files
]
data = {id: path for id, path in zip(id_list, files)}
return data
else:
url = f"http://images.cocodataset.org/zips/{data_type}2014.zip"
file_name = url.split("/")[-1]
if not os.path.exists(self.data_path + f"/{file_name}"):
downloder(url, self.data_path + f"/{file_name}")
path = self.data_path + f"/{data_type}2014_array"
if len(glob.glob(self.data_path +
f"/{data_type}2014/*.jpg")) != data_len:
print(
f"Info:Extract {data_type} images from zip file and convert images to ndarray"
)
if not os.path.exists(path): os.mkdir(path)
import zipfile
with zipfile.ZipFile(self.data_path +
f"/{file_name}") as zip_fp:
file_count = len(zip_fp.filelist)
for i, item in enumerate(zip_fp.filelist):
file_name = item.filename.split("/")[-1].split(".")[0]
zip_fp.extract(item, self.data_path)
if file_name != "":
arr = self.path2array(
self.data_path +
f"/{data_type}2014/{file_name}.jpg")
np.save(path + f"/{file_name}", arr)
progress(i + 1, file_count)
print("")
files = glob.glob(path + "/*.npy")
id_list = [
int(path.split("_")[-1].split(".")[0]) for path in files
]
data = {id: path for id, path in zip(id_list, files)}
return data
def index_templates(self):
posts = []
total_cnt = len(self.statics_tpl)
print("Index %d templates" % total_cnt)
for page_key, page in self.statics_tpl.items():
with open("templates/" + page['tpl_file']) as p:
text = clean_string(p.read())
posts.append({
"post_id": "static-" + page_key,
"title": page['title'],
"text": text
})
for i, post in enumerate(posts):
suffix = post["title"][:25]
progress(i, total_cnt, suffix)
self.index_document(self.index_name, post)
print("\nFinished indexing templates")
def load_depth(self, image_data):
if os.path.exists(self.data_path + "/depth_data.plk"):
with open(self.data_path + "/depth_data.plk", "rb") as fp:
depth_data = pickle.load(fp)
else:
print("Info:convert image to depth")
depth_data = dict()
key_list = list(image_data.keys())
for i, key in enumerate(key_list):
image = image_data[key]
depth_128 = self.image2depth(image["x_256"])
depth_64 = np.resize(depth_128, (64, 64))
depth_data[key] = {"x_128": depth_128, "x_64": depth_64}
progress(i + 1, len(key_list))
with open(self.data_path + "/depth_data.plk", "wb") as fp:
pickle.dump(depth_data, fp)
return depth_data
def get_next_char_with_timing(current_token_chars):
session = requests.Session()
historic_charset_timing = {}
for i in xrange(NUM_SAMPLES):
current_charset_timing = {}
shuffled_charset = CHARSET[:]
random.shuffle(shuffled_charset)
for current_char in shuffled_charset:
test_token = current_token_chars + current_char + CHARSET[0] * (VALID_TOKEN_LEN - 1)
response = session.get(URL, headers={'Authorization': 'Token %s' % test_token})
current_charset_timing[current_char] = float(response.headers['X-Runtime'])
progress(i, NUM_SAMPLES)
ranked_charset_timing = rank_charset_timing(current_charset_timing)
for ichar in ranked_charset_timing:
if ichar in historic_charset_timing:
historic_charset_timing[ichar].append(ranked_charset_timing[ichar])
else:
historic_charset_timing[ichar] = [ranked_charset_timing[ichar]]
average_char_ranking = {}
for ichar in historic_charset_timing:
average_char_ranking[ichar] = numpy.mean(historic_charset_timing[ichar])
avg_items = average_char_ranking.items()
avg_items.sort(value_sort)
found_char = avg_items[0][0]
print('Found character "%s"' % found_char)
pprint.pprint(avg_items)
pprint.pprint(average_char_ranking)
return found_char
def load_images(self):
files_cont = len(glob.glob(self.data_path + f"/array/{self.image_size[0]}/*.npy"))
if files_cont == image_size:
files = glob.glob(self.data_path + f"/array/{self.image_size[0]}/*.npy")
id_list = [int(path.split("_")[-1].split(".")[0]) for path in files]
data = {id:path for id, path in zip(id_list,files)}
else:
url = "http://www.vision.caltech.edu.s3-us-west-2.amazonaws.com/visipedia-data/CUB-200-2011/CUB_200_2011.tgz"
title = "CUB_200_2011.tgz"
if not os.path.exists(self.data_path + f"/{title}"):downloder(url, self.data_path + f"/{title}")
if len(glob.glob(self.data_path + "/image/*/*.jpg")) != image_size:
print("Info:Extracting image data from tar file")
import tarfile, shutil
with tarfile.open(self.data_path + f"/{title}", 'r') as tar_fp:
tar_fp.extractall(self.data_path)
shutil.move(self.data_path + "/CUB_200_2011/images", self.data_path)
os.rename(self.data_path + "/images", self.data_path + "/image")
files = glob.glob(self.data_path + "/image/*/*.jpg")
path = self.data_path + "/array"
if not os.path.exists(path):os.mkdir(path)
path = path + f"/{self.image_size[0]}"
if not os.path.exists(path):
os.mkdir(path)
print("Info:Conveting image data path to ndarray")
for i, _path in enumerate(files):
file_name = _path.split("/")[-1].split(".")[0]
arr = self.path2array(_path)
np.save(path + f"/{file_name}.npy", arr)
progress(i+1, image_size)
print("")
files = glob.glob(path + "/*.npy")
id_list = [int(path.split("_")[-1].split(".")[0]) for path in files]
data = {id:path for id, path in zip(id_list,files)}
return data
def load_images(self):
if os.path.exists(self.data_path + "/image_data.plk"):
with open(self.data_path + "/image_data.plk", "rb") as fp:
data = pickle.load(fp)
else:
file_name = "102flowers.tgz"
url = "http://www.robots.ox.ac.uk/~vgg/data/flowers/102/102flowers.tgz"
if not os.path.exists(self.data_path + f"/{file_name}"):
downloder(url, self.data_path + f"/{file_name}")
if len(glob.glob(self.data_path + "/images/*.jpg")) != IMAGE_SIZE:
print("Info:Extracting image data from tar file")
import tarfile
with tarfile.open(self.data_path + f"/{file_name}",
'r') as tar_fp:
tar_fp.extractall(self.data_path)
os.rename(self.data_path + "/jpg", self.data_path + "/images")
data = {}
files = glob.glob(self.data_path + "/images/*.jpg")
print("Info:load image data")
for i, _path in enumerate(files):
arr_id = int(_path.split("_")[-1].split(".")[0])
arr_256 = self.path2array(_path, 256)
arr_128 = self.path2array(_path, 128)
arr_64 = self.path2array(_path, 64)
data[arr_id] = {
"x_256": arr_256,
"x_128": arr_128,
"x_64": arr_64
}
progress(i + 1, IMAGE_SIZE)
print("")
with open(self.data_path + "/image_data.plk", "wb") as fp:
pickle.dump(data, fp)
return data
def load_images(self):
files_cont = len(glob.glob(self.data_path + f"/array/{self.image_size[0]}/*.npy"))
if files_cont == image_size:
files = glob.glob(self.data_path + f"/array/{self.image_size[0]}/*.npy")
id_list = [int(path.split("_")[-1].split(".")[0]) for path in files]
data = {id:path for id, path in zip(id_list,files)}
else:
url = "http://www.robots.ox.ac.uk/~vgg/data/flowers/102/102flowers.tgz"
title = "102flowers.tgz"
if not os.path.exists(self.data_path + f"/{title}"):downloder(url, self.data_path + f"/{title}")
if len(glob.glob(self.data_path + "/image/*.jpg")) != image_size:
print("Info:Extracting image data from tar file")
import tarfile
with tarfile.open(self.data_path + f"/{title}", 'r') as tar_fp:
tar_fp.extractall(self.data_path)
os.rename(self.data_path + "/jpg", self.data_path + "/image")
files = glob.glob(self.data_path + "/image/*.jpg")
path = self.data_path + "/array"
if not os.path.exists(path):os.mkdir(path)
path = path + f"/{self.image_size[0]}"
if not os.path.exists(path):
os.mkdir(path)
print("Info:Conveting image data path to ndarray")
for i, _path in enumerate(files):
file_name = _path.split("/")[-1].split(".")[0]
arr = self.path2array(_path)
np.save(path + f"/{file_name}.npy", arr)
progress(i+1, image_size)
print("")
files = glob.glob(path + "/*.npy")
id_list = [int(path.split("_")[-1].split(".")[0]) for path in files]
data = {id:path for id, path in zip(id_list,files)}
return data
features['echoisms'], features['selfish'],#get_echoisms(lyric), get_selfish_degree(lyric),
count_duplicate_lines(lyric), features['is_title_in_lyrics'],# (row['Song'], lyric),
features['rhymes'],#get_rhymes(lyric),
features['verb_tenses']['present'], features['verb_tenses']['past'], features['verb_tenses']['future'], #verb_freq['present'], verb_freq['past'], verb_freq['future'],
freq['ADJ'], freq['ADP'], freq['ADV'], freq['AUX'], freq['CONJ'],
freq['CCONJ'], freq['DET'], freq['INTJ'], freq['NOUN'], freq['NUM'],
freq['PART'], freq['PRON'], freq['PROPN'], freq['PUNCT'], freq['SCONJ'],
freq['SYM'], freq['VERB'], freq['X'], freq['SPACE'],
# Sentiment stuff
sentiment[0], sentiment[1],
song[2]
)
rows.append(elem)
count += 1
progress(count, total, '{}/{}'.format(count, total))
df = pd.DataFrame(rows)
df.columns = ['ARTIST', 'SONG_TITLE', 'LYRICS_VECTOR', 'TITLE_VECTOR',
'LINE_COUNT', 'WORD_COUNT', 'ECHOISMS', 'SELFISH_DEGREE',
'DUPLICATE_LINES', 'IS_TITLE_IN_LYRICS', 'RHYMES', 'VERB_PRESENT',
'VERB_PAST', 'VERB_FUTURE', 'ADJ_FREQUENCIES', 'CONJUCTION_FREQUENCIES',
'ADV_FREQUENCIES', 'AUX_FREQUENCIES', 'CONJ_FREQUENCIES', 'CCONJ_FREQUENCIES',
'DETERMINER_FREQUENCIES', 'INTERJECTION_FREQUENCIES', 'NOUN_FREQUENCIES',
'NUM_FREQUENCIES', 'PART_FREQUENCIES', 'PRON_FREQUENCIES', 'PROPN_FREQUENCIES',
'PUNCT_FREQUENCIES', 'SCONJ_FREQUENCIES', 'SYM_FREQUENCIES', 'VERB_FREQUENCIES',
'X_FREQUENCIES', 'SPACE_FREQUENCIES',
'SENTIMENT', 'SUBJECTIVITY',
'EMOTION'
]
try:
from instrumental.drivers.sourcemeasureunit.hp import HP_4156C
SOURCEMETER = HP_4156C(visa_address='GPIB0::17::INSTR')
except:
print('no sourcemeter available. exiting.')
exit()
else:
print('HP opened')
SOURCEMETER.set_channel(channel=2)
SOURCEMETER.set_current_compliance(Q_(100e-6, 'A'))
bring_to_breakdown(SOURCEMETER, Vbd)
for t in range(round(bias_settle_time)):
progress(t + 1,
round(bias_settle_time),
status='Bias settle wait {}/{:g} sec'.format(
t + 1, bias_settle_time))
time.sleep(1.0)
# Start measurements
timestamp = []
measurements = []
bias_current = []
maxlevel = []
minlevel = []
temperatures = []
print('\nPerforming measurement...')
for i in range(num_measures):
progress(i + 1, num_measures, status='Running measurement')
import song_featurize as sf
import utils.progress as progress
import pandas as pd
ML4Q_PATH = './datasets/silver_standard/songsDF.csv'
df = pd.read_csv(ML4Q_PATH)
X = list()
total = len(df)
i = 0
for idx, row in df.iterrows():
progress.progress(i, total, 'Parsed songs')
point = sf.featurize(row['PID'], row['Artist'], row['Title'])
if point is not None:
point.append(row['SeqID'])
X.append(point)
i += 1
df = pd.DataFrame(
X,
columns=[
'PLAYLIST_PID', 'ARTIST', 'SONG_TITLE', 'LYRICS_VECTOR',
'TITLE_VECTOR', 'LINE_COUNT', 'WORD_COUNT', 'ECHOISMS',
'SELFISH_DEGREE', 'DUPLICATE_LINES', 'IS_TITLE_IN_LYRICS', 'RHYMES',
'VERB_PRESENT', 'VERB_PAST', 'VERB_FUTURE', 'ADJ_FREQUENCIES',
'CONJUCTION_FREQUENCIES', 'ADV_FREQUENCIES', 'AUX_FREQUENCIES',
'CONJ_FREQUENCIES', 'CCONJ_FREQUENCIES', 'DETERMINER_FREQUENCIES',
'INTERJECTION_FREQUENCIES', 'NOUN_FREQUENCIES', 'NUM_FREQUENCIES',
'PART_FREQUENCIES', 'PRON_FREQUENCIES', 'PROPN_FREQUENCIES',
def main():
conf = get_args()
dataset = MSCOCO(conf)
VOC_SIZE = dataset.jp_voc_size if conf.use_lang == "jp" else dataset.en_voc_size
SAMPLE_SIZE = conf.sample_size // conf.gpu_num if conf.gpu_num > 1 else conf.sample_size
SEQ_LEN = conf.seq_len_jp if conf.use_lang == "jp" else conf.seq_len_en
index2tok = dataset.jp_index2tok if conf.use_lang == "jp" else dataset.en_index2tok
if not conf.silent:
save_path = os.path.abspath(script_path + conf.save_path)
if not os.path.exists(save_path): os.mkdir(save_path)
save_path = os.path.abspath(save_path + f"/{conf.use_lang}")
if not os.path.exists(save_path): os.mkdir(save_path)
preview_path = os.path.abspath(save_path + "/preview")
if not os.path.exists(preview_path): os.mkdir(preview_path)
netG, netD = build_models(conf, VOC_SIZE, SEQ_LEN)
optimizerG, optimizerD = build_optimizer(netG, netD, conf.adam_lr,
conf.adam_beta1, conf.adam_beta2)
pprog = print_progress(conf.pre_gen_max_epoch, conf.batch_size,
dataset.train_data_len)
updater = Updater(netG, netD, optimizerG, optimizerD, conf)
def pretrain_generatr():
print("==========================================")
print("Info:start genarator pre train")
pre_gen_loss_hist = np.zeros((1, conf.pre_gen_max_epoch),
dtype="float32")
for i in range(conf.pre_gen_max_epoch):
count = 0
total_loss = 0
start = time.time()
for data in dataset.get_data():
data = toGPU(data, conf.gpu_num)
loss = updater.update_pre_gen(data)
total_loss += loss.data.cpu().numpy()
count += 1
if dataset.now_iter % conf.display_interval == 0:
elapsed = time.time() - start
pprog(elapsed, dataset.get_state)
start = time.time()
pre_gen_loss_hist[0, i] = total_loss / count
if not conf.silent:
data = dataset.sample(conf.sample_size)
sample_generate(netG, data, SAMPLE_SIZE, index2tok, conf.gpu_num,\
conf.noise_dim, preview_path + f"/sample_text_pretrain.txt")
np.save(save_path + "/pre_gen_loss_hist", pre_gen_loss_hist)
torch.save(netG.state_dict(), save_path + "/pretrain_gen_params")
print("\n\n\n\n==========================================")
def pretrain_discriminator():
print("==========================================")
print("Info:start discriminator pre train")
dataset.clear_state()
pprog.max_iter = conf.pre_dis_max_epoch
pre_dis_hist = np.zeros((4, conf.pre_dis_max_epoch), dtype="float32")
for i in range(conf.pre_dis_max_epoch):
count = 0
total_loss = 0
total_real_acc = 0
total_fake_acc = 0
total_wrong_acc = 0
start = time.time()
for data in dataset.get_data():
data = toGPU(data, conf.gpu_num)
loss, real_acc, fake_acc, wrong_acc = updater.update_dis(data)
total_loss += loss.data.cpu().numpy()
total_real_acc += real_acc.data.cpu().numpy()
total_fake_acc += fake_acc.data.cpu().numpy()
total_wrong_acc += wrong_acc.data.cpu().numpy()
count += 1
if dataset.now_iter % conf.display_interval == 0:
elapsed = time.time() - start
pprog(elapsed, dataset.get_state)
start = time.time()
pre_dis_hist[0, i] = total_loss / count
pre_dis_hist[1, i] = total_real_acc / count
pre_dis_hist[2, i] = total_fake_acc / count
pre_dis_hist[3, i] = total_wrong_acc / count
if not conf.silent:
np.save(save_path + "/pre_dis_hist", pre_dis_hist)
torch.save(netD.state_dict(), save_path + "/pretrain_dis_params")
print("\n\n\n\n==========================================")
if os.path.exists(save_path + "/pretrain_gen_params"):
netG.load_state_dict(torch.load(save_path + "/pretrain_gen_params"))
else:
pretrain_generatr()
if os.path.exists(save_path + "/pretrain_dis_params"):
netD.load_state_dict(torch.load(save_path + "/pretrain_dis_params"))
else:
pretrain_discriminator()
print("==========================================")
print("Info:start main train")
dataset.clear_state()
pprog.max_iter = conf.max_epoch
train_loss_hist = np.zeros((5, conf.max_epoch), dtype="float32")
val_loss_hist = np.zeros((5, conf.max_epoch), dtype="float32")
val_count = dataset.val_data_len // conf.batch_size
if dataset.val_data_len % conf.batch_size != 1: val_count += 1
for i in range(conf.max_epoch):
#train loop
count = 1
total_g_loss = 0
total_d_loss = 0
total_real_acc = 0
total_fake_acc = 0
total_wrong_acc = 0
start = time.time()
for p in netG.parameters():
p.requires_grad = True
for p in netD.parameters():
p.requires_grad = True
for data in dataset.get_data():
data = toGPU(data, conf.gpu_num)
if count % conf.n_dis == 0:
loss = updater.update_PG(data)
total_g_loss += loss.data.cpu().numpy()
loss, real_acc, fake_acc, wrong_acc = updater.update_dis(data)
total_d_loss += loss.data.cpu().numpy()
total_real_acc += real_acc.data.cpu().numpy()
total_fake_acc += fake_acc.data.cpu().numpy()
total_wrong_acc += wrong_acc.data.cpu().numpy()
count += 1
if dataset.now_iter % conf.display_interval == 0:
elapsed = time.time() - start
pprog(elapsed, dataset.get_state)
start = time.time()
train_loss_hist[0, i] = total_d_loss / count
train_loss_hist[1, i] = total_real_acc / count
train_loss_hist[2, i] = total_fake_acc / count
train_loss_hist[3, i] = total_wrong_acc / count
train_loss_hist[4, i] = total_g_loss / (count // 5)
print("\n\n\n")
#val loop
print(f"Validation {i+1} / {conf.max_epoch}")
count = 0
total_g_loss = 0
total_d_loss = 0
total_real_acc = 0
total_fake_acc = 0
total_wrong_acc = 0
start = time.time()
for p in netG.parameters():
p.requires_grad = False
for p in netD.parameters():
p.requires_grad = False
for data in dataset.get_data(is_val=True):
data = toGPU(data, conf.gpu_num)
g_loss, d_loss, real_acc, fake_acc, wrong_acc = updater.evaluate(
data)
count += 1
if dataset.now_iter % conf.display_interval == 0:
elapsed = time.time() - start
progress(count + 1, val_count, elapsed)
progress(count, val_count, elapsed)
val_loss_hist[0, i] = total_d_loss / count
val_loss_hist[1, i] = total_real_acc / count
val_loss_hist[2, i] = total_fake_acc / count
val_loss_hist[3, i] = total_wrong_acc / count
val_loss_hist[4, i] = total_g_loss / (count // 5)
print("\u001B[5A", end="")
if (i + 1) % conf.snapshot_interval == 0 and not conf.silent:
data = dataset.sample(conf.sample_size)
sample_generate(netG, data, SAMPLE_SIZE, index2tok, conf.gpu_num,\
conf.noise_dim, preview_path + f"/sample_text_{i+1:04d}.txt")
np.save(save_path + "/train_loss_hist", train_loss_hist)
np.save(save_path + "/val_loss_hist", val_loss_hist)
torch.save(netG.state_dict(),
save_path + f"/gen_params_{i+1:04d}.pth")
torch.save(netD.state_dict(),
save_path + f"/dis_params_{i+1:04d}.pth")
if not conf.silent:
np.save(save_path + "/train_loss_hist", train_loss_hist)
np.save(save_path + "/val_loss_hist", val_loss_hist)
data = dataset.sample(conf.sample_size)
sample_generate(netG, data, SAMPLE_SIZE, index2tok, conf.gpu_num,\
conf.noise_dim, preview_path + "/sample_text.txt")
torch.save(netG.state_dict(), save_path + "/gen_params.pth")
torch.save(netD.state_dict(), save_path + "/dis_params.pth")
print("\n\n\n\n==========================================")
print("Info:finish train")
def send_requests(db, known_valid, test_case_1, test_case_2, missing_chars):
"""
:param db: The database where samples are stored
:param known_valid: The known valid characters
:param test_case_1: One character to test (test case 1)
:param test_case_2: One character to test (test case 2)
:param missing_chars: The total number of chars of the API key
:return: None. All is stored in the DB
"""
session = requests.Session()
http_adapter = requests.adapters.HTTPAdapter(max_retries=3)
session.mount('http://', http_adapter)
session.mount('https://', http_adapter)
token_test_case_1 = generate_test_token(known_valid,
test_case_1,
missing_chars)
token_test_case_2 = generate_test_token(known_valid,
test_case_2,
missing_chars)
print('Collecting %s samples for:' % NUM_SAMPLES * 2)
print(' - %s' % token_test_case_1)
print(' - %s' % token_test_case_2)
print('')
print('Test name: %s' % TEST_NAME)
for i in xrange(NUM_SAMPLES):
#
# What I'm trying to do here is to get timings in pairs.
# https://github.com/andresriancho/django-rest-framework-timing/issues/5
#
tmp_results = {}
# Sending the HTTP requests in different order during sample capture is
# something recommended by Paul McMillan and Sebastian Schinzel, they
# recommend it because it might break some caches
shuffled_token_tests = [(0, token_test_case_1),
(1, token_test_case_2)]
random.shuffle(shuffled_token_tests)
for j, token in shuffled_token_tests:
response, naive_time = send_with_naive_timing(session, URL, token)
tmp_results[j] = (response, naive_time, token)
data = {'test_name': TEST_NAME,
'capture_timestamp': time.time()}
for j, (response, naive_time, token) in enumerate(tmp_results.values()):
data.update({'x_runtime_%s' % j: response.headers['X-Runtime'],
'userspace_rtt_microseconds_%s' % j: naive_time,
'token_%s' % j: token})
db.insert(data)
if i % (NUM_SAMPLES / 1000) == 0:
progress(i, NUM_SAMPLES)
log.write('\n')
def download_lyric(song):
try:
lyric = lyricwikia.get_lyrics(song['Artist'], song['Title'])
filename = '_'.join([song['Mood'], song['Artist'], song['Title']])
filename = filename.replace('/', '-') # The '/' should never appear
with open(os.path.join(args.output, filename), 'w') as sfile:
sfile.write(lyric)
return True
except lyricwikia.LyricsNotFound:
err('Could not download {}: {}, {}'.format(song['Index'], song['Artist'], song['Title']))
return False
if __name__ == '__main__':
# Get the number of songs we are going to download
totalTitles = songs_count(args.input)
# Create output directory
create_output_dir(args.output)
# Download songs
count = 0
errCount = 0
for lyric in lyric_entries_generator(args.input):
progress(count, totalTitles, 'Errors encountered: {}'.format(errCount))
if not download_lyric(lyric):
errCount += 1
count += 1
