def __init__(self):
self.logger = Logger().get_logger()
def sleep(self, seconds):
self.logger = Logger(loggername=self.classname).get_logger()
time.sleep(seconds)
self.logger.info("Sleep for {} seconds".format(seconds))
import sys
import os
import shutil
import subprocess
sys.path.append('../')
from utils.log import Logger
basedir = os.path.split(os.path.realpath(__file__))[0]
lockfile = os.path.join(os.path.dirname(basedir),'lock','install_base.lck')
logger = Logger(logname=os.path.join(os.path.dirname(basedir),'logs','install.log'),loglevel=
1,logger='install_base').getlog()
base_list = ['gcc','g++','make','zip','unzip','lrzsz','python-pip','python-dev']
def install_base():
for base_pkg in base_list:
child = subprocess.Popen(['apt-get','-y','install',base_pkg],stdout=subprocess.PIPE,stderr=subprocess.PIPE)
(out,err) = child.communicate()
logger.info(out)
if err:
logger.error(err)
def run():
if os.path.exists(lockfile):
print "You can only run this part once time"
sys.exit(1)
else:
logger.info('++++++++++++++ start install_base ++++++++++++++++++')
install_base()
os.mknod(lockfile)
logger.info('++++++++++++++ end install_base ++++++++++++++++++')
ap50_95, ap50 = evaluator.evaluate(self.yolov4)
logger.info('ap50_95:{}|ap50:{}'.format(ap50_95, ap50))
writer.add_scalar('val/COCOAP50', ap50, epoch)
writer.add_scalar('val/COCOAP50_95', ap50_95, epoch)
self.__save_model_weights(epoch, ap50)
print('save weights done')
else:
assert print('dataset must be VOC or COCO')
end = time.time()
logger.info(" ===cost time:{:.4f}s".format(end - start))
logger.info("=====Training Finished. best_test_mAP:{:.3f}%====".format(self.best_mAP))
if __name__ == "__main__":
global logger, writer
parser = argparse.ArgumentParser()
parser.add_argument('--weight_path', type=str, default='weight/mobilenetv3.pth', help='weight file path')#weight/darknet53_448.weights
parser.add_argument('--resume', action='store_true',default=False, help='resume training flag')
parser.add_argument('--gpu_id', type=int, default=-1, help='whither use GPU(eg:0,1,2,3,4,5,6,7,8) or CPU(-1)')
parser.add_argument('--log_path', type=str, default='log/', help='log path')
parser.add_argument('--accumulate', type=int, default=2, help='batches to accumulate before optimizing')
parser.add_argument('--fp_16', type=bool, default=False, help='whither to use fp16 precision')
opt = parser.parse_args()
writer = SummaryWriter(logdir=opt.log_path + '/event')
logger = Logger(log_file_name=opt.log_path + '/log.txt', log_level=logging.DEBUG, logger_name='YOLOv4').get_log()
Trainer(weight_path=opt.weight_path,
resume=opt.resume,
gpu_id=opt.gpu_id,
accumulate=opt.accumulate,
fp_16=opt.fp_16).train()
def main():
global args
set_random_seed(args.seed)
if not args.use_avai_gpus:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices
use_gpu = torch.cuda.is_available()
if args.use_cpu:
use_gpu = False
sys.stdout = Logger(osp.join(args.save_dir, "log.txt"))
if use_gpu:
print('Currently using GPU {}'.format(args.gpu_devices))
cudnn.benchmark = True
else:
warnings.warn(
'Currently using CPU, however, GPU is highly recommended')
print('Initializing image data manager')
dm = ImageDataManager(use_gpu, **trainset_kwargs(args))
trainloader, testloader_dict = dm.return_dataloaders()
print('Initializing model: {}'.format(args.arch))
model = models.init_model(name=args.arch,
num_classes=dm.num_train_pids,
loss={'xent', 'htri'},
pretrained=not args.no_pretrained,
use_gpu=use_gpu)
print('Model size: {:.3f} M'.format(count_num_param(model)))
if args.load_weights and check_isfile(args.load_weights):
load_pretrained_weights(model, args.load_weights)
model = nn.DataParallel(model).cuda() if use_gpu else model
criterion_xent = CrossEntropyLoss(num_classes=dm.num_train_pids,
use_gpu=use_gpu,
label_smooth=args.label_smooth)
criterion_htri = TripletLoss(margin=args.margin)
optimizer = init_optimizer(model, **optimizer_kwargs(args))
scheduler = init_lr_scheduler(optimizer, **lr_scheduler_kwargs(args))
if args.resume and check_isfile(args.resume):
args.start_epoch = resume_from_checkpoint(args.resume,
model,
optimizer=optimizer)
time_start = time.time()
print('=> Start training')
for epoch in range(args.start_epoch, args.max_epoch):
train(epoch, model, criterion_xent, criterion_htri, optimizer,
trainloader, use_gpu)
scheduler.step()
if (epoch + 1) > args.start_eval and args.eval_freq > 0 and (
epoch + 1) % args.eval_freq == 0 or (epoch +
1) == args.max_epoch:
print('=> Validation')
print('Evaluating {} ...'.format(args.test_set))
queryloader = testloader_dict['query']
galleryloader = testloader_dict['test']
rank1 = test(model, queryloader, galleryloader, use_gpu)
save_checkpoint(
{
'state_dict': model.state_dict(),
'rank1': rank1,
'epoch': epoch + 1,
'arch': args.arch,
'optimizer': optimizer.state_dict(),
}, args.save_dir)
elapsed = round(time.time() - time_start)
elapsed = str(datetime.timedelta(seconds=elapsed))
print('Elapsed {}'.format(elapsed))
default='null',
help="Server codec for parameter averaging")
parse.add_argument("--workers",
type=str,
default='worker.json',
help='Worker list file, json type')
parse.add_argument("--network-bandwidth",
dest='bandwidth',
type=int,
default=100,
help='Network bandwidth in Mbps.')
arg = parse.parse_args()
logger = Logger(title_info='P-SGD User Submit', log_to_file=True)
logger.log_message('Initializing with parameters: ')
logger.log_message('\t --model <file name {}>'.format(arg.model_file))
logger.log_message('\t --node_count <node count {}>'.format(arg.n))
logger.log_message('\t --batch_size <batch size {}>'.format(arg.b))
logger.log_message('\t --redundancy <r {}>'.format(arg.r))
logger.log_message(
'\t --codec <communication codec and protocol {}>'.format(arg.codec))
logger.log_message(
'\t --optimizer <optimizer for model training {}>'.format(arg.op))
logger.log_message('\t --epochs <training epochs {}>'.format(arg.epochs))
logger.log_message('\t --dataset <using {}>'.format(arg.dataset))
logger.log_message('\t --non-iid <{}>'.format(arg.make_iid_dataset))
logger.log_message('\t --gradient-descent <Gradient method {}>'.format(
parser.add_argument('--dataset', type=str, default='one-billion-words')
args = parser.parse_args()
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
args.n_gpu = len(args.gpu.split(','))
model_path = '../checkpoints/xlnet_maskedlm/{}'.format(args.dataset)
log_path = '../logs/xlnet_maskedlm'
args.model_path = model_path
args.log_path = log_path
if not os.path.exists(log_path):
os.makedirs(log_path)
log_file = '{}/{}.log'.format(log_path, args.dataset)
print('The log file is ', log_file)
logger = Logger(log_file)
logger.logger.info(args)
if not os.path.exists(model_path):
os.makedirs(model_path)
try:
# load the pre-trained model and tokenizer
tokenizer = XLNetTokenizer.from_pretrained(args.model_path)
model = XLNetLMHeadModel.from_pretrained(args.model_path)
logger.logger.info('Initialize XLNet from checkpoint {}.'.format(
args.model_path))
except:
tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased')
model = XLNetLMHeadModel.from_pretrained('xlnet-base-cased')
logger.logger.info('Initialize XLNet with default parameters.')
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import tornado
import tornado.web
import tornado.ioloop
import traceback
from utils.thread_pool import GThreadPool
from utils.log import Logger
logger = Logger("server")
class BaseHandler(tornado.web.RequestHandler):
@tornado.web.asynchronous
def get(self, *args):
"""
args是从route正则中匹配出来的参数
"""
GThreadPool.add_job(self._call_back, self.handler, *args)
def _call_back(self, *args):
try:
func = args[0][0]
tpargs = args[0][1:]
func(*tpargs)
message = "%s %s (%s)" % (self.request.method, self.request.uri,
self.request.remote_ip)
logger.info(message)
except:
traceback.print_exc()
# -*- coding: utf-8 -*-
from utils.log import Logger
from global_ import global_cls
from utils.rwExcel import writeExcel
import time
mylogger = Logger(logger='Assertion').getlog()
errors = []
# 实际值与预期值的对比 如果实际值与预期值相等则pass，否则fail
def verifyEquals(web_tools, actual, expected, title):
mylogger.info(title)
try:
assert actual == expected
mylogger.info('pass!!!')
except Exception as e:
errors.append(e)
mylogger.error('fail: ' + " '" + actual + "' != '" + expected)
web_tools.screen_shot(title)
# 实际值与预期值的对比 如果实际值与预期值不等则pass，否则fail
def verifyNotEquals(web_tools, actual, expected, title):
mylogger.info(title)
try:
assert actual != expected
mylogger.info('pass!!!')
except Exception as e:
errors.append(e)
mylogger.error('fail: ' + " '" + actual + "' == '" + expected)
def setUpClass(self):
self.START_TIME = time.time()
self.logger = Logger(CASE_NAME).get_logger()
self.driver = get_driver()
self.dr = Basic(self.driver, self.logger)
self.p = ShelfPage(self.dr)
import redis
from utils.redissub import RedisSub
from utils.redisvalue import RemoteRedis
import utils.exprv
from components.autohost import AutoHost
from utils.log import FileLogger, Logger
import components.hub as hub
import components.hosthandler as hosthandler
DEBUG = True
if not DEBUG:
logger = FileLogger('expserver.txt')
else:
logger = Logger()
exp_list = {}
rc = redis.Redis()
db = torndb.Connection("localhost", 'exp', user='******', password='******')
class Server(Greenlet):
def __init__(self, channel='experiment'):
super(Server, self).__init__()
self.redis = rc
self.sub = RedisSub(channel, self.on_message)
def run(self):
# -*- encoding: utf-8 -*-
import os
import json
import time
import signal
import traceback
from multiprocessing import (Value, Queue)
from config import (PROJECT_DIR, LOG_FILE, CAFFE_MODEL, CAFFE_DEPLOY,
LABEL_FILE, REDIS_JOB_KEY, MAX_PREDICT_PROCESS)
from utils import process
from utils.log import Logger
from utils.redis_tools import RedisTool
from specimen.revert import revert_image, is_symbol
from classifier import RecEngine
log = Logger(LOG_FILE).logger
QUEUE_SIZE = 5000
SLEEP_TIME = 0.001
class Agent(object):
"""
OCR识别引擎
"""
def __init__(self):
self._stop = Value('d', 0)
self.caffe_model = os.path.join(PROJECT_DIR, CAFFE_MODEL)
self.caffe_deploy = os.path.join(PROJECT_DIR, CAFFE_DEPLOY)
self.label_file = os.path.join(PROJECT_DIR, LABEL_FILE)
def add_trigger(request):
"""创建触发器视图"""
if request.method == 'GET':
form_obj = trigger_form.AddTriggerForm()
return render(request, 'add_trigger.html', {'form_obj': form_obj})
elif request.method == 'POST':
applications_id_list = request.POST.getlist('applications_id')
items_id_list = request.POST.getlist('items_id')
specified_item_key_list = request.POST.getlist('specified_item_key')
operator_list = request.POST.getlist('operator')
threshold_list = request.POST.getlist('threshold')
logic_with_next_list = request.POST.getlist('logic_with_next')
data_calc_func_list = request.POST.getlist('data_calc_func')
data_calc_func_args_list = request.POST.getlist('data_calc_func_args')
count = 0
while count < len(applications_id_list):
if applications_id_list[count] == '' and items_id_list[
count] == '' and threshold_list[count] == '':
applications_id_list.pop(count)
items_id_list.pop(count)
specified_item_key_list.pop(count)
operator_list.pop(count)
threshold_list.pop(count)
logic_with_next_list.pop(count)
data_calc_func_list.pop(count)
data_calc_func_args_list.pop(count)
if count == 0:
pass
else:
count -= 1
else:
count += 1
trigger_data = {
'name': request.POST.get('name'),
'templates_id': request.POST.get('templates_id'),
'severity': request.POST.get('severity'),
'enabled': request.POST.get('enabled'),
'memo': request.POST.get('memo')
}
form_obj = trigger_form.AddTriggerForm(trigger_data)
if form_obj.is_valid():
try:
with transaction.atomic():
trigger_obj = models.Trigger.objects.create(
**form_obj.cleaned_data)
triggers_id = trigger_obj.id
for index in range(len(applications_id_list)):
trigger_expression_data = {
'triggers_id': triggers_id,
'applications_id': applications_id_list[index],
'items_id': items_id_list[index],
'specified_item_key':
specified_item_key_list[index],
'operator': operator_list[index],
'threshold': threshold_list[index],
'logic_with_next': logic_with_next_list[index],
'data_calc_func': data_calc_func_list[index],
'data_calc_func_args':
data_calc_func_args_list[index]
}
if trigger_expression_data['threshold'] == '': # 无阈值
if trigger_expression_data[
'applications_id'] != '' or trigger_expression_data[
'items_id'] != '': # 应用集或监控项不为空
raise Exception('触发器表达式有误，有应用集或监控项，但无阈值')
elif trigger_expression_data['threshold'] != '': # 有阈值
if trigger_expression_data[
'applications_id'] == '' or trigger_expression_data[
'items_id'] == '': # 应用集或监控项为空
raise Exception('触发器表达式有误，有阈值，但无对应应用集或监控项')
else:
if trigger_expression_data[
'data_calc_func_args'] != '':
json.loads(trigger_expression_data[
'data_calc_func_args'])
if len(applications_id_list
) == index + 1: # 最后一个表达式
if trigger_expression_data[
'logic_with_next']:
raise Exception(
'触发器表达式有误，最后一个表达式不能有逻辑关系符号')
models.TriggerExpression.objects.create(
**trigger_expression_data)
Logger().log(message='创建触发器成功,%s' % trigger_obj.name,
mode=True)
return redirect('/monitor_web/trigger.html')
except Exception as e:
Logger().log(message='创建触发器失败,%s' % str(e), mode=False)
raise ValidationError(_('添加触发器失败'), code='invalid')
else:
return render(request, 'add_trigger.html', {'form_obj': form_obj})
def __init__(self, node_id, offset):
super().__init__(node_id, offset)
# wait
self.__log = Logger('ParaServer'.format(node_id), log_to_file=True)
self.__done: [bool] = False
self.__transfer: [ITransfer] = None
def click_search_btn(self):
self.rec_search_btn().click()
self.logger = Logger(loggername=self.__class__.__name__).get_logger()
self.logger.debug(u'点击"立即登录 "按钮.')
def main():
tic = int(time()) # Identify the program process by the timestamp
parser = AP()
parser.add_argument('--config', type=str, help='Gym environment')
opt = parser.parse_args()
# Load configuration files as dictionaries.
with open('configs/default.yaml') as f:
default = yaml.load(f)
try:
with open(opt.config) as f:
cfg = yaml.load(f)
ParseConfig(cfg, default)
except:
cfg = default
print(
'No valid configuration file exists. Use default configurations.')
# Set configurations
env_name = cfg['ENV']
agent_name = cfg['AGENT'].upper()
brain = cfg['BRAIN']
params = cfg['PARAMETERS']
networks = cfg['NETWORKS']
backend = networks['BACKEND']
networks = networks['NETWORKS']
gamma = params['GAMMA']
theta = float(params['THETA'])
lr = float(params['LEARNING_RATE'])
eps_start = float(params['EPSILON_START'])
eps_decay = float(params['EPSILON_DECAY'])
eps_type = params['EPSILON_DECAY_TYPE']
eps_end = float(params['EPSILON_END'])
max_epoch = int(float(params['MAX_EPOCH']))
max_step = int(float(params['MAX_STEP']))
num_episode = int(float(params['NUM_EPISODE']))
capacity = int(float(params['MEMORY_CAPACITY']))
batch_size = int(float(params['BATCH_SIZE']))
update = int(params['TARGET_UPDATE'])
funcs = cfg['FUNCTIONS']
vis = funcs['VIS']
intv = funcs['INTERVAL']
eval_flag = funcs['EVAL']
memorize = funcs['MEMORIZE']
logdir = funcs['LOGDIR']
plot_flag = funcs['PLOT']
verbose = funcs['VERBOSE']
# Setup program logger
if logdir and logdir[-1] != '/':
logdir += '/'
main_logger = Logger('Main',
logdir=logdir,
logfile='Main{}.log'.format(tic))
main_logger('Program timestamp: {}'.format(tic))
main_logger('Argument configurations:')
main_logger(cfg)
env = gym.make(env_name)
main_logger('Environment Description:')
main_logger('Environment type: {}'.format(env.class_name()))
main_logger('Observation space: {}'.format(env.observation_space))
main_logger('Action space: {}'.format(env.action_space))
if agent_name == 'RANDOM':
agent = RLAgents.core.RandomAgent(env=env)
elif agent_name == 'POLICYITERATION':
agent = RLAgents.PI.PolicyIterationAgent(env=env,
brain=brain,
theta=theta,
models=brain,
gamma=gamma)
logfile = 'PILearn{}.log'.format(tic)
elif agent_name == 'VALUEITERATION':
agent = RLAgents.VI.ValueIterationAgent(env=env,
brain=brain,
theta=theta,
models=brain,
gamma=gamma)
logfile = 'VILearn{}.log'.format(tic)
elif agent_name == 'SARSA':
agent = RLAgents.SARSA.SARSAAgent(env=env,
brain=brain,
epsilon=eps_start,
epsilon_decay_type=eps_type,
epsilon_decay=eps_decay,
epsilon_end=eps_end,
lr=lr,
gamma=gamma)
logfile = 'SARSALearn{}.log'.format(tic)
elif agent_name == 'QLEARNING':
agent = RLAgents.QL.QLearningAgent(env=env,
brain=brain,
epsilon=eps_start,
epsilon_decay_type=eps_type,
epsilon_decay=eps_decay,
epsilon_end=eps_end,
lr=lr,
gamma=gamma)
logfile = 'SARSALearn{}.log'.format(tic)
elif agent_name == 'DQN':
agent = RLAgents.DQN.GeneralDQNAgent(env=env,
gamma=gamma,
brain=brain,
capacity=capacity,
max_step=max_step,
double=False,
epsilon=eps_start,
epsilon_decay=eps_decay,
epsilon_decay_type=eps_type,
epsilon_end=eps_end,
network=networks[0],
batch_size=batch_size,
update=update,
backend=backend,
verbose=verbose)
logfile = 'DQNLearn{}.log'.format(tic)
elif agent_name == 'DDQN':
agent = RLAgents.DQN.GeneralDQNAgent(env=env,
gamma=gamma,
brain=brain,
capacity=capacity,
max_step=max_step,
double=True,
epsilon=eps_start,
epsilon_decay=eps_decay,
epsilon_decay_type=eps_type,
epsilon_end=eps_end,
network=networks[0],
batch_size=batch_size,
update=update,
backend=backend,
verbose=verbose)
logfile = 'DQNLearn{}.log'.format(tic)
elif agent_name == 'DDPG':
agent = RLAgents.DDPG.DDPGAgent(env=env,
gamma=gamma,
brain=brain,
capacity=capacity,
max_step=max_step,
networks=networks,
batch_size=batch_size,
update=update,
backend=backend,
verbose=verbose)
logfile = 'DDPGLearn{}.log'.format(tic)
else:
raise ValueError('The agent is not supported at this moment.')
if max_epoch > 0:
learn_logger = Logger('MainLearn', logdir=logdir, logfile=logfile)
print('Start learning...')
agent.learn(max_epoch=max_epoch, eval=eval_flag, logger=learn_logger)
toc = int(time())
main_logger('Learning takes {}s'.format(toc - tic))
print('Start evaluation...')
_ = agent.render(num_episode=num_episode,
vis=vis,
intv=intv,
logger=main_logger)
if memorize:
print('Saving learned models...')
agent.save_brain(tic)
main_logger('The model is saved.')
if logdir and eval_flag and plot_flag: # Plot can only be drawn when learning is evaluated and logged.
main_logger('Plot episodic reward of learning process...')
plot_learn(logdir + logfile)
toc = int(time())
main_logger('Program takes {}s'.format(toc - tic))
Date: 2021/6/13
Describe:
"""
import json
import requests
import os
import sys
from utils.log import Logger
log_filename = os.path.realpath(sys.argv[1]).split("\\")[-1]. \
replace(".py::", "_").replace("::", ".") \
if sys.argv[1] else None
# log_filename = os.path.basename(__file__)
print(f"日志主文件路径={log_filename}")
print(f"Sys.argv参数列表={sys.argv}")
logger = Logger(log_filename).logger
class BaseApi:
def __init__(self):
self.logger = logger
def request(self, request: dict):
if request.get("protocol") == "rpc":
return self.rpc_request(**request)
if request.get("protocol") == "tcp":
return self.tcp_request(**request)
import time
from selenium.common.exceptions import NoSuchElementException
import os.path
from utils.log import Logger
from utils.config import DRIVER_PATH, REPORT_PATH
# create a logger instance
logger = Logger().get_logger()
class BasePage(object):
'''
configuration public method
'''
def __init__(self, driver):
self.driver = driver
def quit_browser(self):
self.driver.quit()
def forward(self):
self.driver.forward()
logger.info("Click forward button on current page")
def back(self):
self.driver.back()
# 隐式等待
def wait(self, seconds):
self.driver.implicitly_wait(seconds)
# -*- coding=utf-8 -*-
import time
import unittest
from selenium import webdriver
from api.send_request import SendRequest
from utils.config import Config, DRIVER_PATH, TestData, ENV, PLATFORM, CHROME_DRIVER_FILE
from os import path
from utils.log import Logger
logger = Logger(__name__).get()
class BaseTest(unittest.TestCase):
def setUp(self):
self.config = Config()
self.test_data = TestData(env=ENV)
self.options = webdriver.ChromeOptions()
if PLATFORM == 'H5':
# 模拟移动设备
self.mobile_emulation = {'deviceName': 'iPhone 6'}
self.options.add_experimental_option("mobileEmulation",
self.mobile_emulation)
# for error:exited abnormally
self.options.add_argument('--no-sandbox')
# python3.x 方式
import importlib
importlib.reload(sys)
from utils.db import MyConnection
from utils.log import SysLogger as Logger
# debug开关
DEBUG = False
'''
日志初使化
采用两种日志类型：stdout 和 syslog，线上系统采用syslog日志
stdout日志使用场景：研发环境的调试、NBE系统
'''
logger = Logger('tornado', Logger.DEBUG_LEVEL)
# mysql database configuration
db = MyConnection(host='192.168.1.72:3306',
database='test',
user='',
password='',
logger=logger)
# 返回给ajax的数据
return_msg = dict(success=dict(err=0, status='00000', msg='', seqid=''),
error=dict(err=1, status=-1, msg=u'系统错误', seqid=''))
# 数据库ping的时间
db_ping_seconds = 1000
def main(
problem,
popsize,
algorithm,
save_freq,
loss_type=['trickLogD', 'minimax', 'ls'],
postfix=None,
nPassD=1, #backpropagation pass for discriminator
batchSize=64,
metric="default",
output_dir="runs",
gradients_penalty=False):
if not (problem in problem_table.keys()):
exit(-1)
#task
task_args = problem_table[problem][1]
task = problem_table[problem][0](nPassD, popsize, batchSize, metric)
net_otype = task.net_output_type()
# description
description_name = '{}_{}_{}_{}'.format(
str(task),
algorithm,
popsize,
postfix if postfix is not None else "",
)
# share params
nloss = len(loss_type)
lr = task_args['lr'] # initial learning rate for adam G
lrd = task_args['lrd'] # initial learning rate for adam D
b1 = task_args['b1'] # momentum term of adam
beta = task_args['beta'] # momentum term of adam
samples = task_args['metric_samples'] # metric samples
DIM = task_args['dim'] # momentum term of adam
GP_norm = gradients_penalty # if use gradients penalty on discriminator
LAMBDA = 2. # hyperparameter sudof GP
# algorithm params
if algorithm == "egan":
VARIATION = "all"
MULTI_OBJECTIVE_SELECTION = False
elif algorithm == "moegan":
VARIATION = "all"
MULTI_OBJECTIVE_SELECTION = True
elif algorithm == "smoegan":
VARIATION = "deepqlearning"
MULTI_OBJECTIVE_SELECTION = True
else:
exit(-2)
# Load the dataset
def create_generator_trainer(noise=None,
discriminator=None,
lr=0.0002,
b1=0.5,
DIM=64):
return GeneratorTrainer(noise, task.create_geneator(noise, DIM),
discriminator, lr, b1)
# MODEL D
print("Building model and compiling functions...")
# Prepare Theano variables for inputs and targets
real_imgs = net_otype('real_imgs')
fake_imgs = net_otype('fake_imgs')
# Create neural network model
discriminator = task.create_discriminator(DIM, GP_norm)
# Create expression for passing real data through the discriminator
real_out = lasagne.layers.get_output(discriminator, real_imgs)
# Create expression for passing fake data through the discriminator
fake_out = lasagne.layers.get_output(discriminator, fake_imgs)
# Create loss expressions
discriminator_loss = (
lasagne.objectives.binary_crossentropy(real_out, 1) +
lasagne.objectives.binary_crossentropy(fake_out, 0)).mean()
# Gradients penalty norm
if GP_norm is True:
alpha = t_rng.uniform((batchSize, 1), low=0., high=1.)
differences = fake_imgs - real_imgs
interpolates = real_imgs + (alpha * differences)
gradients = theano.grad(lasagne.layers.get_output(
discriminator, interpolates).sum(),
wrt=interpolates)
slopes = T.sqrt(T.sum(T.sqr(gradients), axis=(1)))
gradient_penalty = T.mean((slopes - 1.)**2)
D_loss = discriminator_loss + LAMBDA * gradient_penalty
b1_d = 0.
else:
D_loss = discriminator_loss
b1_d = 0.
# Create update expressions for training
discriminator_params = lasagne.layers.get_all_params(discriminator,
trainable=True)
lrtd = theano.shared(lasagne.utils.floatX(lrd))
updates_d = lasagne.updates.adam(D_loss,
discriminator_params,
learning_rate=lrtd,
beta1=b1_d)
#lrt = theano.shared(lasagne.utils.floatX(lr))
# Fd Socre
Fd = theano.gradient.grad(discriminator_loss, discriminator_params)
Fd_score = beta * T.log(sum(T.sum(T.sqr(x)) for x in Fd))
# max is ~7.5 for toy dataset and ~0.025 for real ones (it will be updated after 1 iteration, which is likely the worst one)
Fd_auto_normalization = AutoNormalization(float(0.1))
# Compile a function performing a training step on a mini-batch (by giving
# the updates dictionary) and returning the corresponding training loss:
train_d = theano.function([real_imgs, fake_imgs],
discriminator_loss,
updates=updates_d)
# Compile another function generating some data
dis_fn = theano.function([real_imgs, fake_imgs],
[fake_out.mean(), Fd_score])
disft_fn = theano.function([real_imgs, fake_imgs], [
real_out.mean(),
fake_out.mean(), (real_out > 0.5).mean(),
(fake_out > 0.5).mean(), Fd_score
])
#main MODEL G
noise = T.matrix('noise')
generator_trainer = create_generator_trainer(noise, discriminator, lr, b1,
DIM)
# Finally, launch the training loop.
print("Starting training...")
print(description_name)
#build dirs
path_front, path_logs, path_models, path_models_last, path_images = build_output_dirs(
output_dir, description_name)
#define a problem instance
instances = []
instances_old = []
#generator of a offspring
def generate_offsptring(xreal, loss_id, pop_id, inst=None):
if inst == None:
newparams = create_generator_trainer(noise=noise,
discriminator=discriminator,
lr=lr,
b1=b1,
DIM=DIM).get()
inst = Instance(-float("inf"), float("inf"), newparams, -1, pop_id,
None)
#init gen
generator_trainer.set(inst.params)
#train
generator_trainer.train(loss_type[loss_id], task.noise_batch())
#score
xfake = generator_trainer.gen(task.noise_batch())
frr_score, fd_score = dis_fn(xreal, xfake)
#new instance
new_instance = Instance(frr_score, fd_score, generator_trainer.get(),
loss_id, pop_id, xfake)
#save
instances.append(new_instance)
#info stuff
return new_instance
#init varation
variation = get_varation(VARIATION)(popsize, nloss, generate_offsptring)
#reval pop with new D
def reval_pupulation(in_instances):
#ret
out_instances = []
#generates new batches of images for each generator, and then eval these sets by means (new) D
for inst in in_instances:
generator_trainer.set(inst.params)
xfake = generator_trainer.gen(task.noise_batch())
frr_score, fd_score = dis_fn(xreal_eval, xfake)
out_instances.append(
Instance(frr_score,
fd_score,
generator_trainer.get(),
inst.loss_id,
inst.pop_id,
xfake,
im_parent=True))
return out_instances
#log stuff
LOG_HEADER, LOG_TEMPLATE = build_log_template(popsize, nloss)
log = Logger(os.path.join(path_logs, 'logs.tsv'),
header=LOG_HEADER.encode())
timer = Timer()
losses_counter = [0] * nloss
# We iterate over epochs:
for n_updates in task.get_range():
#get batch
xmb = task.batch()
#get eval batch
if xmb.shape[0] == batchSize:
xreal_eval = xmb
else:
xreal_eval = shuffle(xmb)[:batchSize]
# initial G cluster
if MULTI_OBJECTIVE_SELECTION:
instances_old = reval_pupulation(instances)
else:
instances_old = instances
#reset
instances = []
variation.update(instances_old, task.is_last())
for pop_id in range(0, popsize):
variation.gen(xreal_eval,
instances_old[pop_id] if n_updates else None, pop_id)
if popsize <= (len(instances) + len(instances_old)):
if MULTI_OBJECTIVE_SELECTION == True:
#add parents in the pool
instances = [*instances_old, *instances]
#from the orginal code, we have to maximize D(G(X)),
#Since in NSGA2 performences a minimization,
#We are going to minimize -D(G(X)),
#also we want maximize the diversity score,
#So, we are going to minimize -diversity score (also we wanna normalize that value)
cromos = {
idx:
[-float(inst.fq), -float(Fd_auto_normalization(inst.fd))]
for idx, inst in enumerate(instances)
} # S2
cromos_idxs = [idx for idx, _ in enumerate(instances)]
finalpop = nsga_2_pass(popsize, cromos, cromos_idxs)
instances = [instances[p] for p in finalpop]
with open(os.path.join(path_front, 'last.tsv'),
'wb') as ffront:
for inst in instances:
ffront.write(
(str(inst.fq) + "\t" + str(inst.fd)).encode())
ffront.write("\n".encode())
elif nloss > 1:
#sort new
instances.sort(key=lambda inst: inst.f()
) #(from the orginal code in github) maximize
#cut best ones
instances = instances[len(instances) - popsize:]
for i in range(0, popsize):
xreal, xfake = task.statistic_datas(instances[i].img)
tr, fr, trp, frp, fdscore = disft_fn(xreal, xfake)
fake_rate = np.array([fr]) if i == 0 else np.append(fake_rate, fr)
real_rate = np.array([tr]) if i == 0 else np.append(real_rate, tr)
fake_rate_p = np.array([frp]) if i == 0 else np.append(
fake_rate_p, frp)
real_rate_p = np.array([trp]) if i == 0 else np.append(
real_rate_p, trp)
FDL = np.array([fdscore]) if i == 0 else np.append(FDL, fdscore)
losses_counter[instances[i].loss_id] += 1
# train D
for xreal, xfake in task.iter_data_discriminator(xmb, instances):
train_d(xreal, xfake)
#show it info
print(n_updates, real_rate.mean(), real_rate_p.mean())
#write logs
log.writeln(
LOG_TEMPLATE.format(n_updates, str(timer), fake_rate.mean(),
real_rate.mean(), *fake_rate, *real_rate, *FDL,
*losses_counter).encode())
#varation logs
variation.logs(path_logs, n_updates, last_iteration=task.is_last())
if (n_updates % save_freq == 0
and n_updates != 0) or n_updates == 1 or task.is_last():
#it same
if task.is_last():
id_name_update = math.ceil(float(n_updates) / save_freq)
else:
id_name_update = math.floor(float(n_updates) / save_freq)
#if is egan, eval only the best one.
if MULTI_OBJECTIVE_SELECTION == True:
instances_to_eval = instances
else:
instances_to_eval = [instances[-1]]
#metric
metric_results = task.compute_metrics(
instances_to_eval,
lambda inst, nz: generator_trainer.set(inst.params).gen(nz),
samples)
#mmd2 output
print(n_updates, "metric:", np.min(metric_results), "id:",
np.argmin(metric_results))
#best
best = np.argmin(metric_results)
worst = np.argmax(metric_results)
np.savez(
os.path.join(path_models, 'dis_%s.npz') % (id_name_update),
*lasagne.layers.get_all_param_values(discriminator))
np.savez(
os.path.join(path_models, 'gen_%s.npz') % (id_name_update),
*instances_to_eval[best].params)
#save best
generator_trainer.set(instances_to_eval[best].params)
xfake_best = generator_trainer.gen(task.noise_batch(samples))
#worst_debug
generator_trainer.set(instances_to_eval[worst].params)
xfake_worst = generator_trainer.gen(task.noise_batch(samples))
#save images
task.save_image(xmb, xfake_best, path_images,
"best_%s" % (id_name_update))
task.save_image(xmb, xfake_worst, path_images,
"worst_%s" % (id_name_update))
#print pareto front
with open(
os.path.join(path_front, '%s.tsv') % (id_name_update),
'wb') as ffront:
for idx in range(len(instances_to_eval)):
ffront.write((str(instances_to_eval[idx].fq) + "\t" +
str(instances_to_eval[idx].fd) + "\t" +
str(metric_results[idx])).encode())
ffront.write("\n".encode())
#save all last models:
if task.is_last():
for key, inst in enumerate(instances_to_eval):
np.savez(
os.path.join(path_models_last, 'gen_%s.npz') % (key),
*inst.params)
else:
redis_session = Operation_Redis()
log_path = ''
if (platform.system() == 'Windows'):
log_path = 'D:\\log\\flask_log\\'
elif (platform.system() == 'Linux'):
log_path = '/home/log/flask_log/'
else:
pass
if (not os.path.exists(log_path)):
os.mkdir(log_path)
if 'log' in sections:
logger = Logger(
log_path + config.get('log', 'filename'),
level=config.get('log', 'level'),
when=config.get('log', 'when'),
interval=config.getint('log', 'interval'),
backCount=config.getint('log', 'backCount'),
).logger
else:
logger = Logger(log_path + 'server.log', when='midnight').logger
if __name__ == '__main__':
mongo_session.drop_collection("2B20_craft")
mongo_session.drop_collection("2B21_craft")
mongo_session.drop_collection("2B22_craft")
mongo_session.drop_collection("2B23_craft")
mongo_session.drop_collection("furnace")
# Flag to use FP16 GPU precision.
parser.add_argument(
"--fp_16",
type=bool,
default=False,
help="whither to use fp16 precision",
)
# Parse the arugments into Option objects.
opt = parser.parse_args()
# Instantiate Tensorboard summary writer to even folder.
writer = SummaryWriter(logdir=opt.log_path + "/event")
# Initialize python logger.
logger = Logger(
log_file_name=opt.log_path + "/log.txt",
log_level=logging.DEBUG,
logger_name="YOLOv4",
).get_log()
# Instantiate the trainer and start the training process.
Trainer(
weight_path=opt.weight_path,
resume=opt.resume,
gpu_id=opt.gpu_id,
accumulate=opt.accumulate,
fp_16=opt.fp_16,
).train()
# coding=utf-8
# author:ss
import requests
from utils.log import Logger
import json
import traceback
from utils.formatVlidation import check_json_format
log = Logger()
# requests.adapters.DEFAULT_RETRIES = 5
class BaseApi(object):
def __init__(self, url, header=None):
self.header = header
self.url = url
def send_request(self, method, data=None, cookies=None):
try:
if method == "get":
req = requests.get(self.url,
headers=self.header,
params=data,
cookies=cookies)
elif method == "post":
if self.header is not None and self.header.get(
"Content-Type") == "application/json":
req = requests.post(self.url,
headers=self.header,
def get_page_title(self):
self.logger = Logger(loggername=self.classname).get_logger()
self.logger.info("Current page title is {}".format(self.driver.title))
return self.driver.title
parser.add_argument('--visiual',
type=str,
default='E:\YOLOV4/test_pic',
help='val data path or None')
parser.add_argument('--mode',
type=str,
default='val',
help='val or det or study')
parser.add_argument('--heatmap',
type=str,
default=False,
help='whither show attention map')
opt = parser.parse_args()
writer = SummaryWriter(logdir=opt.log_val_path + '/event')
logger = Logger(log_file_name=opt.log_val_path + '/log_val.txt',
log_level=logging.DEBUG,
logger_name='CIFAR').get_log()
if opt.mode == 'val':
Evaluation(gpu_id=opt.gpu_id,
weight_path=opt.weight_path,
visiual=opt.visiual,
heatmap=opt.heatmap).val()
if opt.mode == 'det':
Evaluation(gpu_id=opt.gpu_id,
weight_path=opt.weight_path,
visiual=opt.visiual,
heatmap=opt.heatmap).Inference()
else:
Evaluation(gpu_id=opt.gpu_id,
weight_path=opt.weight_path,
def __init__(self, driver):
self.driver = driver
self.classname = self.__class__.__name__
self.logger = Logger(loggername=self.classname).get_logger()
def rec_search_btn(self):
self.logger = Logger(loggername=self.__class__.__name__).get_logger()
self.logger.debug(u'找到百度一下Input.')
return self.find_element(selector='id=>su')
def setup_train(i, corpus, args):
"""Setup training.
Handles CPU, single GPU, and distributed training.
Args:
i: The process index. Since one process per GPU, this is also
the GPU index. For single GPU or CPU this is set to 0.
corpus: The corpus for training.
args: Arguments from argparse and main().
"""
args.device = torch.device(args.device.type, i)
# Find rank among all processes.
args.rank = args.node_rank * args.gpu_per_node + i
log = Logger(i, args.tensorboard_dir)
log.train_add_text('arguments', str(args))
log.valid_add_text('arguments', str(args))
if args.dist:
dist.init_process_group(backend='nccl',
init_method='env://',
world_size=args.world_size,
rank=args.rank)
torch.cuda.set_device(args.rank)
# Initialize model
log("| Loading model...")
model = get_model(corpus.vocab, args)
model.to(args.device)
args.total_param = count_param(model)
if hasattr(model, 'layer_pool'):
args.layer_param = sum(
[count_param(layer) for layer in model.layer_pool])
elif hasattr(model, 'layer'):
args.layer_param = count_param(model.layer)
string = f"| Model:\n{model}\n"
string += f"| Total parameters: {args.total_param}\n"
string += f"| Parameters without embedding and pre-softmax linear: {args.layer_param}"
log(string)
log.train_add_text('arguments', string)
log.valid_add_text('arguments', string)
# Create optimizer and scheduler.
optimizer, scheduler = get_optimizer_scheduler(model, args)
if args.fp16:
print("| Floating point 16 precision setting:\n", end='')
model, optimizer = amp.initialize(model, optimizer, opt_level='O2')
if args.dist:
model = DistributedDataParallel(model,
device_ids=[i],
find_unused_parameters=True)
resume_step = 0
resume_epoch = 0
if args.checkpoint is not None:
log("| Loading checkpoint...")
if args.fp16:
resume_step, resume_epoch = load_checkpoint(
args.checkpoint, args.device, model, optimizer, scheduler, amp)
else:
resume_step, resume_epoch = load_checkpoint(
args.checkpoint, args.device, model, optimizer, scheduler)
def update_dropout(module):
if hasattr(module, 'dropout'):
model.dropout = args.dropout
if hasattr(module, 'attn_dropout'):
model.attn_dropout = args.attn_dropout
model.apply(update_dropout)
else:
model.apply(reset_parameters) # Initialize parameters
# Get DataLoader
log("| Processing data...")
train_loader = get_loader(corpus.train, corpus.vocab, args)
if args.valid is not None:
valid_loader = get_eval_loader(corpus.valid, corpus.vocab, args)
log(f"| Training on {socket.gethostname()} with rank {args.rank}.", True)
def train(step, epoch, best_loss):
model.train()
optimizer.zero_grad()
train_loader.dataset.set_seed(epoch)
log.init_epoch(step, epoch, train_loader.dataset.total_target)
epoch_loss = 0
epoch_num_target = 0
for batch_num, batch in enumerate(train_loader):
# TODO debug
f = batch['feature'].data.numpy()
t = batch['target'].data.numpy()
n = batch['num_target']
vocab = corpus.vocab
# TODO print out data to test
# feat = np.transpose(f)
# for data in feat:
# print(vocab.to_text(data))
# continue
# TODO test dataloading
num_target = sum(batch['num_target'])
epoch_num_target += num_target
log.num_target += num_target
log.batch_size += len(batch['num_target'])
try:
feature = batch['feature'].to(args.device)
target = batch['target'].to(args.device)
assert (target != vocab.pad_idx
).sum() == num_target # TODO remove debug check
loss = model(feature, target)
assert loss.dtype == torch.float32 # TODO remove debug check
batch_loss = loss.item()
epoch_loss += batch_loss
log.loss += batch_loss
loss = loss / num_target
loss = loss / args.step_freq
if args.fp16:
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
except RuntimeError as e:
if 'out of memory' in str(e):
log.oom += 1
print(
f"== Rank {args.rank}: Training out of memory. Skipping this batch. =="
)
# Release memory
if 'scaled_loss' in locals():
del scaled_loss
if 'loss' in locals():
del loss
if 'feature' in locals():
del feature
if 'target' in locals():
del target
for param in model.parameters():
if param.grad is not None:
param.grad = None
if args.cuda:
torch.cuda.empty_cache()
else:
raise e
if (batch_num + 1) % args.step_freq == 0:
step, epoch, best_loss = update(step, epoch, best_loss)
if args.max_step is not None and step >= args.max_step:
break
# Remaining batches that doesn't fit in update freq.
if not args.trim_step and (batch_num + 1) % args.step_freq != 0:
step, epoch, best_loss = update(step, epoch, best_loss)
log.end_epoch(step, epoch)
return step, epoch_loss / epoch_num_target, best_loss
def update(step, epoch, best_loss):
loss_scale = 1
if args.fp16:
loss_scale = amp._amp_state.loss_scalers[0]._loss_scale
# Calculate norm of gradients. For logging.
if args.log_norm:
for name, param in model.named_parameters():
if param.grad is None:
continue
norm = param.grad.data.float().norm().item() / loss_scale
log.train_add_scalar('norm/' + name, norm, step)
# Clip gradient
if args.fp16:
norm = torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer),
args.clip_norm)
else:
norm = torch.nn.utils.clip_grad_norm_(model.parameters(),
args.clip_norm)
log.norm += norm
log.clip_norm += min(args.clip_norm, norm)
optimizer.step()
optimizer.zero_grad()
step += 1
if scheduler is not None:
if step < args.warmup_step:
# Linear warmup
warmup_lr = args.lr * step / args.warmup_step
optimizer.param_groups[0]['lr'] = warmup_lr
else:
scheduler.step()
lr = optimizer.param_groups[0]['lr']
log.train(step, lr, loss_scale)
if args.step_per_save != 0 and step % args.step_per_save == 0:
if i == 0:
path = os.path.join(args.checkpoint_dir,
f'checkpoint-{epoch}-{step}.pt')
save_checkpoint(path, step, epoch, model, optimizer, scheduler,
amp if args.fp16 else None)
copyfile(
path,
os.path.join(args.checkpoint_dir, 'checkpoint_last.pt'))
if args.dist:
dist.barrier()
if args.step_per_valid != 0 and step % args.step_per_valid == 0:
# Eval on validation data.
if args.valid is not None:
best_loss = validate(best_loss)
return step, epoch, best_loss
def evaluate():
model.eval()
total_loss = 0
total_target = 0
total = valid_loader.dataset.total_target
if i == 0:
progress = tqdm(desc="Evaluating", total=total, unit=' token')
for batch in valid_loader:
# TODO debug
f = batch['feature'].data.numpy()
t = batch['target'].data.numpy()
n = batch['num_target']
vocab = corpus.vocab
# TODO print out data to test
# feat = np.transpose(f)
# for data in feat:
# print(vocab.to_text(data))
# continue
# TODO test dataloading
num_target = sum(batch['num_target'])
total_target += num_target
feature = batch['feature'].to(args.device)
target = batch['target'].to(args.device)
loss = model(feature, target)
total_loss += loss.item()
if i == 0:
progress.update(num_target)
if i == 0:
progress.close()
return total_loss / total_target
def validate(best_loss):
with torch.no_grad():
loss = evaluate()
log.valid(loss, step, epoch)
if i == 0 and best_loss > loss:
best_loss = loss
best_path = os.path.join(args.checkpoint_dir, 'checkpoint_best.pt')
save_checkpoint(best_path, step, epoch, model, optimizer,
scheduler, amp if args.fp16 else None)
if args.dist:
dist.barrier()
log.valid_add_scalar('best loss', best_loss / math.log(2), step)
log.valid_add_scalar('best ppl', 2**(best_loss / math.log(2)), step)
return best_loss
step = resume_step
best_loss = math.inf
# Start from epoch 1 or resume from next epoch
for epoch in itertools.count(resume_epoch + 1):
# Train on training data.
step, loss, best_loss = train(step, epoch, best_loss)
if args.max_step is not None and step >= args.max_step:
break
if args.epoch_per_valid != 0 and epoch % args.epoch_per_valid == 0:
# Eval on validation data.
if args.valid is not None:
if args.dist:
dist.barrier()
best_loss = validate(best_loss)
# Saving checkpoint.
if args.epoch_per_save != 0 and epoch % args.epoch_per_save == 0:
if i == 0:
path = os.path.join(args.checkpoint_dir,
f'checkpoint-{epoch}-{step}.pt')
save_checkpoint(path, step, epoch, model, optimizer, scheduler,
amp if args.fp16 else None)
copyfile(
path,
os.path.join(args.checkpoint_dir, 'checkpoint_last.pt'))
if args.dist:
dist.barrier()
# Delete old checkpoints.
if i == 0 and (args.keep_step is not None
or args.keep_epoch is not None):
for filename in os.listdir(args.checkpoint_dir):
if re.match(r'checkpoint-\d+-\d+\.pt', filename):
file_epoch, file_step = re.split(r'[-.]', filename)[1:3]
if args.keep_step is not None and int(file_step) <= (
step - args.keep_step):
os.remove(os.path.join(args.checkpoint_dir, filename))
if args.keep_epoch is not None and int(file_epoch) <= (
epoch - args.keep_epoch):
os.remove(os.path.join(args.checkpoint_dir, filename))
if args.dist:
dist.barrier()
if args.max_epoch is not None and epoch >= args.max_epoch:
break
from utils.config import Config
from utils.model import LGBM, XDeepfm
from utils.utils import *
import pandas as pd
import warnings
from feature_engineer import feature_engineer_sparse_matrix
import gc
import os
import sys
from utils.log import Logger
sys.stdout = Logger("log.txt", sys.stdout)
warnings.filterwarnings('ignore', category=Warning)
def train(train_feature, test_feature, label, load_data):
config = Config
if load_data:
train_feature = pd.read_hdf(config.TRAIN_FEATURE_PATH, key='data')
test_feature = pd.read_hdf(config.TEST_FEATURE_PATH, key='data')
label = pd.read_hdf(config.LABEL_PATH, key='data')
# config.CATEGORY_VARIABLES = [c for c in train_feature.columns if (c not in config.TRUE_NUMERICAL_COLUMNS) & (c.split('_')[0] not in ['cateasnum'])]
config.CATEGORY_VARIABLES = 'auto'
print('%d features are used for training...' % (train_feature.shape[1]))
if config.MODEL == 'lgbm':
model = LGBM(config, train_feature, label, test_feature)
else:
model = XDeepfm(config, train_feature, label, test_feature)
model.k_fold_train()
# model.train()
