def generate_summary_best(model_name):
prompts = [1, 2, 3, 4, 5, 6, 7, 8]
# number of essay in test set
length = [-1, 179, 180, 173, 177, 181, 180, 157, 73]
path = utils.mkpath('pred/{}'.format(model_name))
best_ep = [-1]*9
with open(os.path.join(path, 'summary_best.txt'), 'w+') as f:
f.write('{}\n\n'.format(model_name))
f.write('QWK\n')
f.write('epoch\tprompt\tqwk\n')
qwk_avg = 0
for p in prompts:
qwk_df = pd.read_csv(os.path.join(path, 'qwk_{}_val.csv'.format(
p)), header=None, names=['epoch', 'qwk'])
max_idx = qwk_df['qwk'].idxmax()
best_ep[p] = int(qwk_df.iloc[max_idx].values[0])
qwk_df = pd.read_csv(os.path.join(path, 'qwk_{}_test.csv'.format(
p)), header=None, names=['epoch', 'qwk'])
try:
tmp = qwk_df[qwk_df['epoch'] == best_ep[p]].values
# in case of multiple runs of same epoch, pick one with the best QWK
ep, qwk = tmp[tmp.argmax(axis=0)[-1]]
except:
raise Exception(
'Error: epoch {} of prompt {} not found in test'.format(best_ep[p], p))
f.write('{}\t{}\t{}\n'.format(best_ep[p], p, qwk))
qwk_avg += qwk
f.write('\nRobustness per prompt\n')
r_avg = 0
r_aug_avg = 0
for p in prompts:
robustness_df = pd.read_csv(os.path.join(
path, 'robustness_{}_{}.csv'.format(p, best_ep[p])))
r = (robustness_df['worse_resolved'] -
robustness_df['better_resolved']).values[-1]
f.write('{}\t{}\n'.format(p, r))
r_avg += r
r_aug = (robustness_df['worse_resolved'] -
robustness_df['better_resolved']).values[:-2]/length[p]
r_aug_avg += r_aug
f.write('\nRobustness per augment\n')
r_aug_avg /= 8
for a, r in zip(robustness_df['augment'][:-2], r_aug_avg):
f.write('{}\t{}\n'.format(a, r))
f.write('\n')
f.write('QWK Average:\t{}\n'.format(qwk_avg / 8))
f.write('Robustness Average:\t{}\n'.format(r_avg / 8))
f.write('Robustness Average:\t{}\n'.format(r_aug_avg.mean()))
print('summary generated!')
def parse_tweet(self, tweet):
self.max_id = tweet.id
created_at = self.to_datetime(tweet.created_at)
if created_at < self.last_month:
self.finished = True
path = mkpath(self.tweet_path, mkdate(tweet), tweet.id)
mkdir(path)
f = open(mkpath(path, tweet.id) + '.json', 'w')
f.write(tweet.AsJsonString())
retweet_path = mkpath(path, 'retweets')
mkdir(retweet_path)
for retweet in self.api.GetRetweets(tweet.id):
rt_f = open(mkpath(retweet_path, retweet.id) + '.json', 'w')
rt_f.write(retweet.AsJsonString())
def reset(self):
self.cache = mkpath("%s/archiveCache"%(Specs().s['tmpdir']))
self.adir=Specs().s["archiveDir"]
Debug().p("archive dir %s"%self.adir)
self.archives = []
for a in glob.glob(self.adir+"/*.tgz"):
#Debug().p("a: %s"%a)
self.archives.append(a)
self.randList = random.sample(range(len(self.archives)),len(self.archives))
self.randListIndex = 0
def generate_score(prompt, model_name, epoch, y_true, y_pred, aug_pred, test_df):
path = utils.mkpath('pred/{}'.format(model_name))
df = pd.DataFrame()
df['essay_id'] = test_df['essay_id']
df['essay_set'] = test_df['essay_set']
df['domain1_score'] = y_true
df['test'] = y_pred
for key in aug_pred:
df['test_' + key] = aug_pred[key]
df.to_csv(os.path.join(path, 'score_{}_{}.tsv'.format(prompt, epoch)),
sep='\t', index=False)
return df
def generate_summary(model_name, epoch):
prompts = [1, 2, 3, 4, 5, 6, 7, 8]
# number of essay in test set
length = [-1, 179, 180, 173, 177, 181, 180, 157, 73]
path = utils.mkpath('pred/{}'.format(model_name))
with open(os.path.join(path, 'summary_{}.txt'.format(epoch)), 'w+') as f:
f.write('{} epoch {}\n\n'.format(model_name, epoch))
f.write('QWK\n')
qwk_avg = 0
for p in prompts:
qwk_df = pd.read_csv(os.path.join(path, 'qwk_{}_test.csv'.format(
p)), header=None, names=['epoch', 'qwk'])
qwk = qwk_df[qwk_df['epoch'] == epoch].values[-1, -1]
f.write('{}\t{}\n'.format(p, qwk))
qwk_avg += qwk
f.write('\nRobustness per prompt\n')
r_avg = 0
r_aug_avg = 0
for p in prompts:
robustness_df = pd.read_csv(os.path.join(
path, 'robustness_{}_{}.csv'.format(p, epoch)))
r = (robustness_df['worse_resolved'] -
robustness_df['better_resolved']).values[-1]
f.write('{}\t{}\n'.format(p, r))
r_avg += r
r_aug = (robustness_df['worse_resolved'] -
robustness_df['better_resolved']).values[:-2]/length[p]
r_aug_avg += r_aug
f.write('\nRobustness per augment\n')
r_aug_avg /= 8
for a, r in zip(robustness_df['augment'][:-2], r_aug_avg):
f.write('{}\t{}\n'.format(a, r))
f.write('\n')
f.write('QWK Average:\t{}\n'.format(qwk_avg / 8))
f.write('Robustness Average:\t{}\n'.format(r_avg / 8))
f.write('Robustness Average:\t{}\n'.format(r_aug_avg.mean()))
print('summary generated!')
def generate_robustness(prompt, model_name, epoch, y_true, y_pred, aug_pred):
path = utils.mkpath('pred/{}'.format(model_name))
# y_true = rescale_to_int(y_true, prompt)
y_pred_int = rescale_to_int(y_pred, prompt)
aug_pred_int = {}
wr_t, br_t, w_t, b_t = 0, 0, 0, 0
N = len(y_pred) * len(aug_pred)
print('N :', N)
with open(os.path.join(path, 'robustness_{}_{}.csv'.format(prompt, epoch)), 'w+') as f:
f.write('augment,worse_raw,better_raw,worse_resolved,better_resolved\n')
for key in aug_pred:
aug_pred_int[key] = rescale_to_int(aug_pred[key], prompt)
wr, br, w, b = robustness(
y_pred, aug_pred[key], y_pred_int, aug_pred_int[key])
wr_t += wr
br_t += br
w_t += w
b_t += b
f.write('{},{},{},{},{}\n'.format(key, wr, br, w, b))
f.write('sum,{},{},{},{}\n'.format(wr_t, br_t, w_t, b_t))
f.write('avg,{},{},{},{}\n'.format(wr_t/N, br_t/N, w_t/N, b_t/N))
def main(no_debug=True):
print("Starting...")
solved = {}
for year in range(2000, 3000):
year_path = mkpath(ROOT_PATH, year)
if not isdir(year_path):
continue
# part1 part1 README
solved[year] = [[False, False, False] for _ in range(25)]
# Handle days
for day in range(0, 25):
day_path = mkpath(year_path, "Day {:02d}".format(day + 1))
if not isdir(day_path):
continue
files = [
filename for filename in os.listdir(day_path)
if isfile(mkpath(day_path, filename))
]
solved[year][day] = [
"part1.py" in files, "part2.py" in files, "README.md" in files
]
# ------------------------------------------- Long line warning --------------------------------------------
for filename in files:
if os.path.splitext(filename)[1] == ".py":
with open(mkpath(day_path, filename),
'r',
encoding="utf-8") as file:
for line in file:
if len(line.strip()) > 120:
print(
"Warning: long line detected in {}".format(
mkpath(day_path, filename)))
# -------------------------------- Solution files: Replace tabs with spaces --------------------------------
if no_debug:
if solved[year][day][0]:
replace_tabs(mkpath(day_path, "part1.py"))
if solved[year][day][1]:
replace_tabs(mkpath(day_path, "part2.py"))
# ----------------------------------------------- Day README -----------------------------------------------
if "README.md" in files:
readme_path = mkpath(day_path, "README.md")
with open(readme_path, 'r', encoding="utf-8") as file:
readme = file.read()
# Place non-breaking spaces in markdown `code` tags:
readme = re.sub(r"`[^`\n\t\b\r]+`",
lambda m: m.group(0).replace(' ', chr(0x2007)),
readme)
# Handle "<!-- Execute code: "smth" -->" blocks
if no_debug:
readme = readme_exec(readme, day_path)
with open(readme_path, 'w', encoding="utf-8") as file:
file.write(readme)
# ----------------------------------------- TXT files (input.txt) ------------------------------------------
for filename in files:
if os.path.splitext(filename)[1] in (".txt", ".py", ".md"):
filepath = mkpath(day_path, filename)
with open(filepath, 'r', encoding="utf-8") as file:
file.seek(0, 2)
if file.tell() == 0:
ends_with_newline = True
else:
file.seek(file.tell() - 1, 0)
ends_with_newline = file.read() == '\n'
if not ends_with_newline:
with open(filepath, 'a', encoding="utf-8") as file:
file.write('\n')
# Handle year README and webpage
if no_debug:
gen_year_table(mkpath(year_path, "README.md"), solved[year], year)
gen_year_page(mkpath(ROOT_PATH, "docs", year, "index.html"),
solved[year], year)
# Handle global README and webpage
if no_debug:
gen_global_table(mkpath(ROOT_PATH, "README.md"), solved)
gen_home_page(mkpath(ROOT_PATH, "docs", "index.html"), solved)
EPOCH = args.epoch
BATCH_SIZE = args.bs
MODEL_NAME = args.name
print(args)
print('ALL PROMPTS :', prompts)
print('BATCH SIZE :', BATCH_SIZE)
print('MODEL_NAME :', MODEL_NAME)
print('EPOCH :', EPOCH)
print('-------')
for p in prompts:
print('PROMPT :', p)
weight_path = utils.mkpath('weight/{}/{}'.format(MODEL_NAME, p))
weight = utils.get_weight_at_epoch(weight_path, EPOCH)
if not weight:
print('weight not found')
continue
test_df = data_utils.load_data(p, 'test')
print(test_df.shape)
from keras import backend as K
K.clear_session()
model = models.build_elmo_model_full(p,
only_elmo=False,
use_mask=True,
summary=False)
def makefolders():
mkpath("out/datapack/data/milhaxcustommodel/functions/")
mkpath("minecraft/tags/functions/","out/datapack/data/")
def readme_exec(text, path):
return re.sub(
REGEX["exec_code"],
lambda match: handle_match(mkpath(path, match.group(1)), match), text)
def getImageCache():
path = "%s/imageCache" % (Specs().s['tmpdir'])
return mkpath(path)
def getCacheDir(id):
path = ImageHandler.getImageCache() + "/%d" % id
return mkpath(path)
def getCacheDir(id):
path = "%s/%d" % (ImageHandler.getImageCache(), id)
return mkpath(path)
from sys import path as sys_path
import os
sys_path.append("../")
from readme_exec import count_time
from utils import mkpath
result = []
for year in range(2000, 3000):
for path, folders, files in os.walk(mkpath("../../", year)):
cur_path = os.getcwd()
os.chdir(path)
for filename in files:
if os.path.splitext(filename)[1] != ".py":
continue
with open(filename, 'r', encoding="utf-8") as file:
code = file.read().strip()
time = round(count_time(code))
if time >= 1000:
result.append((mkpath(path, filename), time))
os.chdir(cur_path)
with open("exec_time_result.txt", 'w') as file:
for path, time in result: