def eval_topic(run_name, topic, split_name, condition_prefix):
path_prefix = "./output/ukp/" + run_name
prediction_path = path_prefix + "_" + topic
tfrecord_path = "./data/ukp_tfrecord/{}_{}".format(split_name, topic)
res = get_f1_score(tfrecord_path, prediction_path)
f1_score = res["f1"]
proxy = get_task_manager_proxy()
proxy.report_number(run_name, f1_score, condition_prefix+topic)
def abortion_eval(run_name):
topic = "abortion"
print(run_name)
prediction_path, tfrecord_path = get_pred_tfrecord_path(run_name, topic)
print(prediction_path)
d = get_ranking_metrics(tfrecord_path, prediction_path)
MAP = d['MAP']
proxy = get_task_manager_proxy()
def main(run_name):
path_prefix = "./output/ukp/" + run_name
topic = "abortion"
prediction_path = path_prefix + "_" + topic
tfrecord_path = "./data/ukp_tfrecord_2way/dev_" + topic
res = get_f1_score(tfrecord_path, prediction_path, 2)
f1_score = res["f1"]
proxy = get_task_manager_proxy()
proxy.report_number(run_name, f1_score, "avg F-1")
def __init__(self, working_path: str, max_job: int, job_name: str,
worker_factory):
self.machine = get_local_machine_name()
self.task_manager_proxy = get_task_manager_proxy()
self.max_job = max_job
self.working_path = working_path
self.job_name = job_name
self.out_path = os.path.join(working_path, job_name)
self.worker_factory = worker_factory
exist_or_mkdir(self.out_path)
def main(base_name):
path_prefix = "./output/ukp/" + base_name
run_name = base_name
all_res = {}
for topic in data_generator.argmining.ukp_header.all_topics:
prediction_path = path_prefix + "_" + topic
tfrecord_path = "./data/ukp_tfrecord/dev_" + topic
res = get_f1_score(tfrecord_path, prediction_path)
all_res[topic] = res
avg_f1 = average([res["f1"] for topic, res in all_res.items()])
for topic, res in all_res.items():
print(topic, res["f1"])
print("Avg F1 : ", avg_f1)
proxy = get_task_manager_proxy()
proxy.report_number(run_name, avg_f1, "F-1")
pickle.dump(all_res, open("./output/ukp_score_log/" + run_name, "wb"))
def report_number(r):
try:
field = FLAGS.report_field
if "," in field:
field_list = field.split(",")
else:
field_list = [field]
for field in field_list:
value = float(r[field])
condition = None
if FLAGS.report_condition:
condition = FLAGS.report_condition
proxy = get_task_manager_proxy()
proxy.report_number(FLAGS.run_name, value, condition, field)
except Exception as e:
print(e)
def main():
job_name = sys.argv[1]
proxy = get_task_manager_proxy()
info = proxy.query_job_group_status(job_name)
keys = ['started', 'num_done', 'max_job']
last_key_val = {}
while info['state'] != STATE_DONE:
any_change = False
for key in keys:
if key not in last_key_val or last_key_val[key] != info[key]:
any_change = True
last_key_val[key] = info[key]
if any_change:
print("started/done/max={}/{}/{}".
format(info['started'], info['num_done'], info['max_job']))
time.sleep(10)
info = proxy.query_job_group_status(job_name)
print("JOB {} is done".format(job_name))
def loop(max_job):
last_mask = get_last_mark()
tprint("Last mark : ", last_mask)
task_manager_proxy = get_task_manager_proxy()
machine_name = get_local_machine_name()
def is_machine_busy():
active_jobs = task_manager_proxy.get_num_active_jobs(machine_name)
pending_jobs = task_manager_proxy.get_num_pending_jobs(machine_name)
tprint("{} active {} pending".format(active_jobs, pending_jobs))
return active_jobs + pending_jobs > max_job
no_job_time = 0
while no_job_time < 1200:
# check if there is additional job to run
job_id = last_mask + 1
next_sh_path = get_sh_path_for_job_id(job_id)
if os.path.exists(next_sh_path):
last_scheduled_job_id = get_new_job_id() - 1
remaining_jobs = last_scheduled_job_id - job_id
while is_machine_busy():
tprint(
"Sleeping for jobs to be done. Remaining jobs : {}".format(
remaining_jobs))
time.sleep(10)
execute(job_id)
no_job_time = 0
task_info.set("last_task_id ", task_info.last_task_id + 1)
mark(job_id)
last_mask += 1
time.sleep(5)
else:
no_job_time += 10
tprint("no job time: ", no_job_time)
time.sleep(10)
tprint("Terminating")
def main(name, number, condition, field):
proxy = get_task_manager_proxy()
proxy.report_number(name, number, condition, field)
from taskman_client.task_proxy import get_task_manager_proxy
tmp = get_task_manager_proxy()
r = tmp.get_tpu("v4")
print(r)