# this file is only used for continuous evaluation test! import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi from kpi import DurationKpi imikolov_20_avg_ppl_kpi = CostKpi('imikolov_20_avg_ppl', 0.2, 0) imikolov_20_pass_duration_kpi = DurationKpi('imikolov_20_pass_duration', 0.02, 0, actived=True) imikolov_20_avg_ppl_kpi_card4 = CostKpi('imikolov_20_avg_ppl_card4', 0.2, 0) imikolov_20_pass_duration_kpi_card4 = DurationKpi( 'imikolov_20_pass_duration_card4', 0.03, 0, actived=True) tracking_kpis = [ imikolov_20_avg_ppl_kpi, imikolov_20_pass_duration_kpi, imikolov_20_avg_ppl_kpi_card4, imikolov_20_pass_duration_kpi_card4, ] def parse_log(log): ''' This method should be implemented by model developers. The suggestion:
import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi, DurationKpi, AccKpi train_cost_kpi = CostKpi('train_cost', 0.02, actived=True) test_acc_kpi = AccKpi('test_acc', 0.005, actived=True) train_duration_kpi = DurationKpi('train_duration', 0.02, actived=True) train_acc_kpi = AccKpi('train_acc', 0.005, actived=True) tracking_kpis = [ train_acc_kpi, train_cost_kpi, test_acc_kpi, train_duration_kpi, ]
####this file is only used for continuous evaluation test! from __future__ import absolute_import from __future__ import division from __future__ import print_function import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi, DurationKpi, AccKpi #### NOTE kpi.py should shared in models in some way!!!! d_train_cost_kpi = CostKpi('d_train_cost', 0.05, 0, actived=True, desc='train cost of discriminator') g_train_cost_kpi = CostKpi('g_train_cost', 0.05, 0, actived=True, desc='train cost of generator') train_speed_kpi = DurationKpi('duration', 0.05, 0, actived=True, unit_repr='second', desc='train time used in one GPU card') tracking_kpis = [d_train_cost_kpi, g_train_cost_kpi, train_speed_kpi]
####this file is only used for continuous evaluation test! import os import sys #sys.path.insert(0, os.environ['ceroot']) from kpi import CostKpi, DurationKpi, AccKpi #### NOTE kpi.py should shared in models in some way!!!! train_cost_card1_kpi = CostKpi('train_cost_card1', 0.02, 0, actived=True) test_cost_card1_kpi = CostKpi('test_cost_card1', 0.005, 0, actived=True) train_duration_card1_kpi = DurationKpi('train_duration_card1', 0.06, 0, actived=True) train_cost_card4_kpi = CostKpi('train_cost_card4', 0.01, 0, actived=True) test_cost_card4_kpi = CostKpi('test_cost_card4', 0.005, 0, actived=True) train_duration_card4_kpi = DurationKpi('train_duration_card4', 0.06, 0, actived=True) tracking_kpis = [ train_cost_card1_kpi, test_cost_card1_kpi, train_duration_card1_kpi, train_cost_card4_kpi, test_cost_card4_kpi, train_duration_card4_kpi, ]
# this file is only used for continuous evaluation test! import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi, DurationKpi train_cost_card1_kpi = CostKpi('train_cost_card1', 0.08, 0, actived=True, desc='train cost') train_speed_card1_kpi = DurationKpi('train_speed_card1', 0.08, 0, actived=True, desc='train speed in one GPU card') train_cost_card4_kpi = CostKpi('train_cost_card4', 0.08, 0, actived=True, desc='train cost') train_speed_card4_kpi = DurationKpi('train_speed_card4', 0.3, 0, actived=True, desc='train speed in four GPU card') tracking_kpis = [ train_cost_card1_kpi, train_speed_card1_kpi, train_cost_card4_kpi, train_speed_card4_kpi
# this file is only used for continuous evaluation test! import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi from kpi import DurationKpi from kpi import AccKpi each_pass_duration_cpu1_thread1_kpi = DurationKpi( 'each_pass_duration_cpu1_thread1', 0.08, 0, actived=True) train_acc_cpu1_thread1_kpi = CostKpi('train_acc_cpu1_thread1', 0.08, 0) each_pass_duration_gpu1_kpi = DurationKpi('each_pass_duration_gpu1', 0.08, 0, actived=True) train_acc_gpu1_kpi = CostKpi('train_acc_gpu1', 0.08, 0) each_pass_duration_gpu4_kpi = DurationKpi('each_pass_duration_gpu4', 0.08, 0, actived=True) train_acc_gpu4_kpi = CostKpi('train_acc_gpu4', 0.08, 0) tracking_kpis = [ each_pass_duration_cpu1_thread1_kpi, train_acc_cpu1_thread1_kpi, each_pass_duration_gpu1_kpi, train_acc_gpu1_kpi, each_pass_duration_gpu4_kpi, train_acc_gpu4_kpi, ]
### This file is only used for continuous evaluation test! from __future__ import print_function from __future__ import division from __future__ import absolute_import import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi dcgan_d_train_cost_kpi = CostKpi('dcgan_d_train_cost', 0.02, 0, actived=True, desc='train cost of discriminator') dcgan_g_train_cost_kpi = CostKpi('dcgan_g_train_cost', 0.02, 0, actived=True, desc='train cost of generator') tracking_kpis = [dcgan_d_train_cost_kpi, dcgan_g_train_cost_kpi] def parse_log(log): for line in log.split('\n'): fs = line.strip().split('\t') print(fs) if len(fs) == 3 and fs[0] == 'kpis': kpi_name = fs[1] kpi_value = float(fs[2]) yield kpi_name, kpi_value
####this file is only used for continuous evaluation test! from __future__ import absolute_import from __future__ import division from __future__ import print_function import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi, DurationKpi, AccKpi #### NOTE kpi.py should shared in models in some way!!!! train_acc = AccKpi('train_acc', 0.1, 0, actived=True, desc="train acc") train_loss = CostKpi('train_loss', 0.1, 0, actived=True, desc="train loss") tracking_kpis = [train_acc, train_loss] def parse_log(log): ''' This method should be implemented by model developers. The suggestion: each line in the log should be key, value, for example: " train_cost\t1.0 test_cost\t1.0 train_cost\t1.0 train_cost\t1.0 train_acc\t1.2 "
#### NOTE kpi.py should shared in models in some way!!!! train_acc_top1_kpi = AccKpi('train_acc_top1', 0.02, 0, actived=True, desc='TOP1 ACC') train_acc_top5_kpi = AccKpi('train_acc_top5', 0.02, 0, actived=True, desc='TOP5 ACC') train_cost_kpi = CostKpi('train_cost', 0.02, 0, actived=True, desc='train cost') test_acc_top1_kpi = AccKpi('test_acc_top1', 0.02, 0, actived=True, desc='TOP1 ACC') test_acc_top5_kpi = AccKpi('test_acc_top5', 0.02, 0, actived=True, desc='TOP5 ACC') test_cost_kpi = CostKpi('test_cost', 0.02, 0, actived=True, desc='train cost') train_speed_kpi = DurationKpi('train_speed', 0.05,
####this file is only used for continuous evaluation test! import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi, DurationKpi, AccKpi #### NOTE kpi.py should shared in models in some way!!!! train_acc_kpi = AccKpi('train_precision', 0.005, 0, actived=True) test_acc_kpi = CostKpi('test_precision', 0.005, 0, actived=True) train_duration_kpi = DurationKpi('train_duration', 0.05, 0, actived=True) tracking_kpis = [ train_acc_kpi, test_acc_kpi, train_duration_kpi, ] def parse_log(log): for line in log.split('\n'): fs = line.strip().split('\t') print(fs) if len(fs) == 3 and fs[0] == 'kpis': print("-----%s" % fs) kpi_name = fs[1] kpi_value = float(fs[2]) yield kpi_name, kpi_value
### This file is only used for continuous evaluation test! from __future__ import print_function from __future__ import division from __future__ import absolute_import import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi from kpi import AccKpi conv_train_cost_kpi = CostKpi('conv_train_cost', 0.02, 0, actived=True, desc='train cost') conv_train_acc_kpi = AccKpi('conv_train_acc', 0.02, 0, actived=True, desc='train acc') conv_test_cost_kpi = CostKpi('conv_test_cost', 0.02, 0, actived=True, desc='test cost') conv_test_acc_kpi = AccKpi('conv_test_acc', 0.02, 0, actived=True, desc='test acc')
####this file is only used for continuous evaluation test! import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi, DurationKpi, AccKpi #### NOTE kpi.py should shared in models in some way!!!! train_acc_kpi = AccKpi('train_acc', 0.005, actived=True) train_cost_kpi = CostKpi('train_cost', 0.005, actived=False) train_duration_kpi = DurationKpi('train_duration', 0.05, actived=True) tracking_kpis = [ train_acc_kpi, train_cost_kpi, train_duration_kpi, ] def parse_log(log): for line in log.split('\n'): fs = line.strip().split('\t') print(fs) if len(fs) == 3 and fs[0] == 'kpis': print("-----%s" % fs) kpi_name = fs[1] kpi_value = float(fs[2]) yield kpi_name, kpi_value
import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi, DurationKpi, AccKpi train_avg_ppl_kpi = CostKpi('train_avg_ppl_kpi', 0.2, 0) train_pass_duration_kpi = DurationKpi('train_pass_duration_kpi', 0.2, 0) tracking_kpis = [ train_avg_ppl_kpi, train_pass_duration_kpi, ]
####this file is only used for continuous evaluation test! import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi, DurationKpi, AccKpi #### NOTE kpi.py should shared in models in some way!!!! train_cost_kpi = CostKpi('train_cost', 0.02, 0, actived=True) test_acc_kpi = AccKpi('test_acc', 0.01, 0, actived=False) train_speed_kpi = DurationKpi('train_speed', 0.1, 0, actived=True, unit_repr="s/epoch") train_cost_card4_kpi = CostKpi('train_cost_card4', 0.02, 0, actived=True) test_acc_card4_kpi = AccKpi('test_acc_card4', 0.01, 0, actived=False) train_speed_card4_kpi = DurationKpi('train_speed_card4', 0.1, 0, actived=True, unit_repr="s/epoch") tracking_kpis = [ train_cost_kpi, test_acc_kpi, train_speed_kpi, train_cost_card4_kpi, test_acc_card4_kpi, train_speed_card4_kpi, ] def parse_log(log): ''' This method should be implemented by model developers. The suggestion:
# this file is only used for continuous evaluation test! import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi from kpi import DurationKpi train_loss_card1 = CostKpi('train_loss_card1', 0.03, 0, actived=True) train_loss_card4 = CostKpi('train_loss_card4', 0.03, 0, actived=True) train_duration_card1 = DurationKpi('train_duration_card1', 0.01, 0, actived=True) train_duration_card4 = DurationKpi('train_duration_card4', 0.01, 0, actived=True) tracking_kpis = [ train_loss_card1, train_loss_card4, train_duration_card1, train_duration_card4, ] def parse_log(log): ''' This method should be implemented by model developers.
# this file is only used for continuous evaluation test! import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi from kpi import DurationKpi imikolov_20_avg_ppl_kpi = CostKpi('lstm_language_model_loss', 0.02, 0) imikolov_20_pass_duration_kpi = DurationKpi( 'lstm_language_model_duration', 0.02, 0, actived=True) tracking_kpis = [ imikolov_20_avg_ppl_kpi, imikolov_20_pass_duration_kpi, ] def parse_log(log): ''' This method should be implemented by model developers. The suggestion: each line in the log should be key, value, for example: " train_cost\t1.0 test_cost\t1.0 train_cost\t1.0 train_cost\t1.0
# this file is only used for continuous evaluation test! import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi from kpi import DurationKpi from kpi import AccKpi each_step_duration_simnet_card1 = DurationKpi( 'each_step_duration_simnet_card1', 0.03, 0, actived=True) train_loss_simnet_card1 = CostKpi('train_loss_simnet_card1', 0.01, 0, actived=True) each_step_duration_simnet_card4 = DurationKpi( 'each_step_duration_simnet_card4', 0.02, 0, actived=True) train_loss_simnet_card4 = CostKpi('train_loss_simnet_card4', 0.01, 0, actived=True) tracking_kpis = [ each_step_duration_simnet_card1, train_loss_simnet_card1, each_step_duration_simnet_card4, train_loss_simnet_card4, ] def parse_log(log):
# this file is only used for continuous evaluation test! import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi from kpi import DurationKpi from kpi import AccKpi each_pass_duration_cpu1_thread1_kpi = DurationKpi( 'each_pass_duration_cpu1_thread1', 0.08, 0, actived=True) train_loss_cpu1_thread1_kpi = CostKpi('train_loss_cpu1_thread1', 0.08, 0) train_auc_val_cpu1_thread1_kpi = AccKpi('train_auc_val_cpu1_thread1', 0.08, 0) train_batch_auc_val_cpu1_thread1_kpi = AccKpi( 'train_batch_auc_val_cpu1_thread1', 0.08, 0) each_pass_duration_cpu1_thread8_kpi = DurationKpi( 'each_pass_duration_cpu1_thread8', 0.08, 0, actived=True) train_loss_cpu1_thread8_kpi = CostKpi('train_loss_cpu1_thread8', 0.08, 0) train_auc_val_cpu1_thread8_kpi = AccKpi('train_auc_val_cpu1_thread8', 0.08, 0) train_batch_auc_val_cpu1_thread8_kpi = AccKpi( 'train_batch_auc_val_cpu1_thread8', 0.08, 0) each_pass_duration_cpu8_thread8_kpi = DurationKpi( 'each_pass_duration_cpu8_thread8', 0.08, 0, actived=True) train_loss_cpu8_thread8_kpi = CostKpi('train_loss_cpu8_thread8', 0.08, 0) train_auc_val_cpu8_thread8_kpi = AccKpi('train_auc_val_cpu8_thread8', 0.08, 0) train_batch_auc_val_cpu8_thread8_kpi = AccKpi( 'train_batch_auc_val_cpu8_thread8', 0.08, 0) tracking_kpis = [ each_pass_duration_cpu1_thread1_kpi, train_loss_cpu1_thread1_kpi,
####this file is only used for continuous evaluation test! import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi, DurationKpi, AccKpi #### NOTE kpi.py should shared in models in some way!!!! train_cost_kpi = CostKpi('train_cost', 0.02, 0, actived=False) test_cost_kpi = CostKpi('test_cost', 0.005, 0, actived=False) train_duration_kpi = DurationKpi('train_duration', 0.06, 0, actived=False) tracking_kpis = [ train_cost_kpi, test_cost_kpi, train_duration_kpi, ] def parse_log(log): ''' This method should be implemented by model developers. The suggestion: each line in the log should be key, value, for example: " train_cost\t1.0 test_cost\t1.0
import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi, DurationKpi, AccKpi train_avg_loss_kpi = CostKpi('train_avg_loss', 0.2, 0) train_seq_err_kpi = CostKpi('train_seq_err', 0.2, 0) tracking_kpis = [ train_avg_loss_kpi, train_seq_err_kpi, ]
""" continuous_evaluation.py """ import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi from kpi import DurationKpi lstm_train_cost_kpi = CostKpi('lstm_train_cost', 5, 0) lstm_pass_duration_kpi = DurationKpi('lstm_pass_duration', 0.02, 0, actived=True) lstm_train_cost_kpi_card4 = CostKpi('lstm_train_cost_card4', 0.2, 0) lstm_pass_duration_kpi_card4 = DurationKpi('lstm_pass_duration_card4', 0.02, 0, actived=True) tracking_kpis = [ lstm_train_cost_kpi, lstm_pass_duration_kpi, lstm_train_cost_kpi_card4, lstm_pass_duration_kpi_card4, ]
####this file is only used for continuous evaluation test! import os import sys sys.path.insert(0, os.environ['ceroot']) from kpi import CostKpi, DurationKpi, AccKpi #### NOTE kpi.py should shared in models in some way!!!! train_ppl_card1_kpi = CostKpi('train_ppl_card1', 0.02, 0, actived=True) train_duration_card1_kpi = DurationKpi( 'train_duration_card1', 0.02, 0, actived=True) #train_ppl_card4_kpi = CostKpi('train_ppl_card4', 0.02, 0, actived=True) #train_duration_card4_kpi = DurationKpi( # 'train_duration_card4', 0.02, 0, actived=True) tracking_kpis = [ train_ppl_card1_kpi, train_duration_card1_kpi, ] def parse_log(log): ''' This method should be implemented by model developers. The suggestion: each line in the log should be key, value, for example: " train_cost\t1.0 test_cost\t1.0 train_cost\t1.0
####this file is only used for continuous evaluation test! from __future__ import absolute_import from __future__ import division from __future__ import print_function import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi, DurationKpi, AccKpi #### NOTE kpi.py should shared in models in some way!!!! train_acc1 = AccKpi('train_acc1', 0.01, 0, actived=True, desc="train acc1") train_acc5 = AccKpi('train_acc5', 0.01, 0, actived=True, desc="train acc5") train_loss = CostKpi('train_loss', 0.01, 0, actived=True, desc="train loss") test_acc1 = AccKpi('test_acc1', 0.01, 0, actived=True, desc='test acc1') test_acc5 = AccKpi('test_acc5', 0.01, 0, actived=True, desc='test acc5') test_loss = CostKpi('test_loss', 0.01, 0, actived=True, desc='test loss') #train_speed_kpi = DurationKpi( # 'train_speed', # 0.05, # 0, # actived=True, # unit_repr='seconds/image', # desc='train speed in one GPU card') tracking_kpis = [ train_acc1, train_acc5, train_loss, test_acc1, test_acc5, test_loss ] def parse_log(log): '''
# this file is only used for continuous evaluation test! import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi from kpi import DurationKpi each_pass_duration_card1_kpi = DurationKpi('each_pass_duration_card1', 0.08, 0, actived=True) train_face_loss_card1_kpi = CostKpi('train_face_loss_card1', 0.08, 0) train_head_loss_card1_kpi = CostKpi('train_head_loss_card1', 0.08, 0) each_pass_duration_card4_kpi = DurationKpi('each_pass_duration_card4', 0.08, 0, actived=True) train_face_loss_card4_kpi = CostKpi('train_face_loss_card4', 0.08, 0) train_head_loss_card4_kpi = CostKpi('train_head_loss_card4', 0.08, 0) tracking_kpis = [ each_pass_duration_card1_kpi, train_face_loss_card1_kpi, train_head_loss_card1_kpi, each_pass_duration_card4_kpi, train_face_loss_card4_kpi, train_head_loss_card4_kpi, ] def parse_log(log): ''' This method should be implemented by model developers. The suggestion:
####this file is only used for continuous evaluation test! from __future__ import absolute_import from __future__ import division from __future__ import print_function import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi, DurationKpi, AccKpi #### NOTE kpi.py should shared in models in some way!!!! test_acc = AccKpi('test_acc', 0.001, 0, actived=True, desc="test acc") test_cost = CostKpi('test_cost', 0.001, 0, actived=True, desc='test cost') #train_speed_kpi = DurationKpi( # 'train_speed', # 0.05, # 0, # actived=True, # unit_repr='seconds/image', # desc='train speed in one GPU card') tracking_kpis = [test_acc, test_cost] def parse_log(log): ''' This method should be implemented by model developers. The suggestion: each line in the log should be key, value, for example:
""" continuous_evaluation.py """ import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi from kpi import DurationKpi lstm_train_cost_kpi = CostKpi('lstm_train_cost', 5, 0) lstm_pass_duration_kpi = DurationKpi('lstm_pass_duration', 0.02, 0, actived=True) tracking_kpis = [lstm_train_cost_kpi, lstm_pass_duration_kpi]
### This file is only used for continuous evaluation test! from __future__ import print_function from __future__ import division from __future__ import absolute_import import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi train_cost_kpi = CostKpi('train_cost', 0.02, 0, actived=True, desc='train cost') tracking_kpis = [train_cost_kpi] def parse_log(log): for line in log.split('\n'): fs = line.strip().split('\t') print(fs) if len(fs) == 3 and fs[0] == 'kpis': kpi_name = fs[1] kpi_value = float(fs[2]) yield kpi_name, kpi_value def log_to_ce(log): kpi_tracker = {} for kpi in tracking_kpis: kpi_tracker[kpi.name] = kpi
import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi, DurationKpi, AccKpi test_avg_ppl_kpi = CostKpi('test_avg_ppl_kpi', 0.2, 0) train_pass_duration_kpi = DurationKpi('train_pass_duration_kpi', 0.03, 0, actived=True) test_avg_ppl_kpi_card4 = CostKpi('test_avg_ppl_kpi_card4', 0.05, 0, actived=True) train_pass_duration_kpi_card4 = DurationKpi('train_pass_duration_kpi_card4', 0.03, 0, actived=True) tracking_kpis = [ test_avg_ppl_kpi, train_pass_duration_kpi, test_avg_ppl_kpi_card4, train_pass_duration_kpi_card4, ]
# this file is only used for continuous evaluation test! import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi from kpi import DurationKpi each_pass_duration_card1_kpi = DurationKpi('each_pass_duration_card1', 0.08, 0, actived=True) train_avg_cost_card1_kpi = CostKpi('train_avg_cost_card1', 0.08, 0) train_avg_acc_card1_kpi = CostKpi('train_avg_acc_card1', 0.02, 0) each_pass_duration_card4_kpi = DurationKpi('each_pass_duration_card4', 0.08, 0, actived=True) train_avg_cost_card4_kpi = CostKpi('train_avg_cost_card4', 0.08, 0) train_avg_acc_card4_kpi = CostKpi('train_avg_acc_card4', 0.02, 0) tracking_kpis = [ each_pass_duration_card1_kpi, train_avg_cost_card1_kpi, train_avg_acc_card1_kpi, each_pass_duration_card4_kpi, train_avg_cost_card4_kpi, train_avg_acc_card4_kpi, ]
import os import sys sys.path.append(os.environ['ceroot']) from kpi import CostKpi, DurationKpi train_cost_kpi = CostKpi('train_cost', 0.01) train_duration_kpi = DurationKpi('train_duration', 0.04) tracking_kpis = [ train_cost_kpi, train_duration_kpi, ]