def catalog(self):
Logger().info("Go to the catalog menu")
try:
loginAdminPageScreen().connect(Context().environment.login,
Context().environment.password)
time.sleep(3)
adminPageScreen().go_to_admin_menu('Catalog')
time.sleep(5)
Logger().success("Go to the catalog menu")
except:
raise
def orders(self):
Logger().info("Go to the orders menu")
try:
loginAdminPageScreen().connect(Context().environment.login,
Context().environment.password)
time.sleep(3)
adminPageScreen().go_to_admin_menu('CustomerThreads',
'CustomerThreads_return')
time.sleep(5)
Logger().success("Go to the orders menu")
except:
raise
def __init__(self):
"""
@param test: an instance of commons.Test
"""
self._configuration = Configuration()
if self._logger is None:
self._logger = Logger(name=__name__,
browser_name=self._configuration.browser,
language=self._configuration.language)
self._environmentName = self._configuration.environment
self.generate_environment()
self._browserName = self._configuration.browser
def build_tensorboard(self):
from commons.logger import Logger
self.logger = Logger(self.log_save_path)
class Solver(object):
def __init__(self, data_loader, config):
self.data_loader = data_loader
self.noise_n = config.noise_n
self.G_last_act = last_act(config.G_last_act)
self.D_out_n = config.D_out_n
self.D_last_act = last_act(config.D_last_act)
self.G_lr = config.G_lr
self.D_lr = config.D_lr
self.beta1 = config.beta1
self.beta2 = config.beta2
self.epoch = config.epoch
self.batch_size = config.batch_size
self.D_train_step = config.D_train_step
self.save_image_step = config.save_image_step
self.log_step = config.log_step
self.model_save_step = config.model_save_step
self.model_save_path = config.model_save_path
self.log_save_path = config.log_save_path
self.image_save_path = config.image_save_path
self.use_tensorboard = config.use_tensorboard
self.pretrained_model = config.pretrained_model
self.build_model()
if self.use_tensorboard is not None:
self.build_tensorboard()
if self.pretrained_model is not None:
if len(self.pretrained_model) != 2:
raise "must have both G and D pretrained parameters, and G is first, D is second"
self.load_pretrained_model()
def build_model(self):
self.G = Generator(self.noise_n, self.G_last_act)
self.D = Discriminator(self.D_out_n, self.D_last_act)
self.G_optimizer = torch.optim.Adam(self.G.parameters(), self.G_lr,
[self.beta1, self.beta2])
self.D_optimizer = torch.optim.Adam(self.D.parameters(), self.D_lr,
[self.beta1, self.beta2])
if torch.cuda.is_available():
self.G.cuda()
self.D.cuda()
def build_tensorboard(self):
from commons.logger import Logger
self.logger = Logger(self.log_save_path)
def load_pretrained_model(self):
self.G.load_state_dict(torch.load(self.pretrained_model[0]))
self.D.load_state_dict(torch.load(self.pretrained_model[1]))
def reset_grad(self):
self.G_optimizer.zero_grad()
self.D_optimizer.zero_grad()
def to_var(self, x, volatile=False):
if torch.cuda.is_available():
x = x.cuda()
return Variable(x, volatile=volatile)
def train(self):
bce_loss = nn.BCELoss()
print(len(self.data_loader))
for e in range(self.epoch):
for i, batch_images in enumerate(self.data_loader):
batch_size = batch_images.size(0)
real_x = self.to_var(batch_images)
noise_x = self.to_var(
torch.FloatTensor(noise_vector(batch_size, self.noise_n)))
real_label = self.to_var(
torch.FloatTensor(batch_size).fill_(1.))
fake_label = self.to_var(
torch.FloatTensor(batch_size).fill_(0.))
# train D
fake_x = self.G(noise_x)
real_out = self.D(real_x)
fake_out = self.D(fake_x.detach())
D_real = bce_loss(real_out, real_label)
D_fake = bce_loss(fake_out, fake_label)
D_loss = D_real + D_fake
self.reset_grad()
D_loss.backward()
self.D_optimizer.step()
# Log
loss = {}
loss['D/loss_real'] = D_real.data[0]
loss['D/loss_fake'] = D_fake.data[0]
loss['D/loss'] = D_loss.data[0]
# Train G
if (i + 1) % self.D_train_step == 0:
# noise_x = self.to_var(torch.FloatTensor(noise_vector(batch_size, self.noise_n)))
fake_out = self.D(self.G(noise_x))
G_loss = bce_loss(fake_out, real_label)
self.reset_grad()
G_loss.backward()
self.G_optimizer.step()
loss['G/loss'] = G_loss.data[0]
# Print log
if (i + 1) % self.log_step == 0:
log = "Epoch: {}/{}, Iter: {}/{}".format(
e + 1, self.epoch, i + 1, len(self.data_loader))
for tag, value in loss.items():
log += ", {}: {:.4f}".format(tag, value)
print(log)
if self.use_tensorboard:
for tag, value in loss.items():
self.logger.scalar_summary(
tag, value,
e * len(self.data_loader) + i + 1)
# Save images
if (e + 1) % self.save_image_step == 0:
noise_x = self.to_var(
torch.FloatTensor(noise_vector(32, self.noise_n)))
fake_image = self.G(noise_x)
save_image(
fake_image.data,
os.path.join(self.image_save_path,
"{}_fake.png".format(e + 1)))
if (e + 1) % self.model_save_step == 0:
torch.save(
self.G.state_dict(),
os.path.join(self.model_save_path,
"{}_G.pth".format(e + 1)))
torch.save(
self.D.state_dict(),
os.path.join(self.model_save_path,
"{}_D.pth".format(e + 1)))
import sys
import os
import time, traceback
import json
PACKAGE_ROOT = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.path.insert(0, PACKAGE_ROOT)
from commons.logger import Logger
from config.initializers import KAFKA_BROKER, KAFKA_TOPIC, KAFKA_CONSUMER_GROUP_ID
from kafka import KafkaConsumer
logger = Logger().get_logger("test")
class Consumer(object):
def __init__(self):
"""Tries to establing the Kafka consumer connection"""
try:
logger.debug("Creating new kafka consumer using brokers: " +
KAFKA_BROKER + ' and topic ' + KAFKA_TOPIC)
self.kafka_consumer = KafkaConsumer(
group_id=KAFKA_CONSUMER_GROUP_ID,
bootstrap_servers=KAFKA_BROKER,
consumer_timeout_ms=8000)
self.kafka_consumer.subscribe([KAFKA_TOPIC])
except KeyError as e:
logger.error('Missing setting named ' + str(e),
{'ex': traceback.format_exc()})
except:
logger.error("Couldn't initialize kafka consumer for topic",
{'ex': traceback.format_exc()})
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import time
import unittest
import sys
import os
sys.path.append(os.path.split(os.path.dirname(__file__))[0])
from commons.browser_engine import BrowserEngine
from pages.baidu_homepage import HomePage
from commons.logger import Logger
log = Logger(logger='baidusearch').get_log()
class BaiduSearch(unittest.TestCase):
def setUp(self):
browser = BrowserEngine(self)
self.driver = browser.open_browser(self)
def tearDown(self):
self.driver.quit()
def test_baidu_search(self):
homepage = HomePage(self.driver)
homepage.type_search('selenium')
homepage.submit_btn()
time.sleep(2)
homepage.get_screenshot()
assert 'selenium' in homepage.get_page_title()
log.info('test pass.')
PACKAGE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.path.insert(0,PACKAGE_ROOT)
import json
import time, traceback
import logging
import websocket
import threading
from commons.logger import Logger
from commons.hit_counter import HitCounter
from commons.redis_connector import RedisConnector
from websocket import create_connection
from core.producer.producer import Producer
from config.initializers import REDIS_STREAM_KEY
logger = Logger().get_logger("test")
transaction_counter = HitCounter()
time_counter=0
def stream():
ws = websocket.WebSocket()
ws = create_connection('wss://ws.blockchain.info/inv')
ws.send(json.dumps({"op":"unconfirmed_sub"}))
return ws
def persist_in_redis(r, stream=REDIS_STREAM_KEY):
try:
global time_counter
time_counter += 1
data = { time_counter: transaction_counter.get_hit() }
r.xadd(stream, data)
from time import sleep
import unittest
from pages.login_page import *
from pages.index_page import IndexPage
from cases.basecase import BaseCase
from pages.know_page import KnowPage
from commons.tool import *
from commons.logger import Logger
logger = Logger().logger
class RanKnow(BaseCase):
'''
知识排序
'''
def test_rank_know(self):
'''
排序_验证通过修改时间排序；CRM-ST-BG-013
:return:
'''
# username = "******"
# password = "******"
#
# # 登录
# sleep(4)
# lp = LoginPage(self.driver)
# sleep(2)
# lp.open()
# lp.login(username, password)
# sleep(2)
# # 去知识页面
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import configparser
import os
from commons.logger import Logger
log = Logger(logger='config_read').get_log()
def get_config(section, key):
try:
config_file = os.path.split(os.path.dirname(__file__))[0] + '/config/config.ini'
config = configparser.ConfigParser()
config.read(config_file, encoding='utf-8')
return config.get(section, key)
except Exception as e:
log.error('config read fail', format(e))
import os
import sys
import json
from kafka import KafkaProducer
from commons.logger import Logger
from config.initializers import KAFKA_BROKER, KAFKA_TOPIC
PACKAGE_ROOT = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.path.insert(0, PACKAGE_ROOT)
logger = Logger().get_logger("test")
class Producer(object):
def __init__(self, topic=KAFKA_TOPIC):
self.producer = KafkaProducer(
bootstrap_servers=KAFKA_BROKER,
value_serializer=lambda m: json.dumps(m).encode('ascii'))
self.topic = topic
def push(self, json_data):
self.producer.send(self.topic, json_data)
class Solver(object):
def __init__(self, data_loader, config):
self.data_loader = data_loader
self.noise_n = config.noise_n
self.G_last_act = last_act(config.G_last_act)
self.D_out_n = config.D_out_n
self.D_last_act = last_act(config.D_last_act)
self.G_lr = config.G_lr
self.D_lr = config.D_lr
self.beta1 = config.beta1
self.beta2 = config.beta2
self.epoch = config.epoch
self.batch_size = config.batch_size
self.D_train_step = config.D_train_step
self.save_image_step = config.save_image_step
self.log_step = config.log_step
self.model_save_step = config.model_save_step
self.clip_value = config.clip_value
self.lambda_gp = config.lambda_gp
self.model_save_path = config.model_save_path
self.log_save_path = config.log_save_path
self.image_save_path = config.image_save_path
self.use_tensorboard = config.use_tensorboard
self.pretrained_model = config.pretrained_model
self.build_model()
if self.use_tensorboard is not None:
self.build_tensorboard()
if self.pretrained_model is not None:
if len(self.pretrained_model) != 2:
raise "must have both G and D pretrained parameters, and G is first, D is second"
self.load_pretrained_model()
def build_model(self):
self.G = Generator(self.noise_n, self.G_last_act)
self.D = Discriminator(self.D_out_n, self.D_last_act)
self.G_optimizer = torch.optim.Adam(self.G.parameters(), self.G_lr,
[self.beta1, self.beta2])
self.D_optimizer = torch.optim.Adam(self.D.parameters(), self.D_lr,
[self.beta1, self.beta2])
if torch.cuda.is_available():
self.G.cuda()
self.D.cuda()
def build_tensorboard(self):
from commons.logger import Logger
self.logger = Logger(self.log_save_path)
def load_pretrained_model(self):
self.G.load_state_dict(torch.load(self.pretrained_model[0]))
self.D.load_state_dict(torch.load(self.pretrained_model[1]))
def denorm(self, x):
out = (x + 1) / 2
return out.clamp_(0, 1)
def reset_grad(self):
self.G_optimizer.zero_grad()
self.D_optimizer.zero_grad()
def to_var(self, x, volatile=False):
if torch.cuda.is_available():
x = x.cuda()
return Variable(x, volatile=volatile)
def train(self):
print(len(self.data_loader))
for e in range(self.epoch):
for i, batch_images in enumerate(self.data_loader):
batch_size = batch_images.size(0)
label = torch.FloatTensor(batch_size)
real_x = self.to_var(batch_images)
noise_x = self.to_var(
torch.FloatTensor(noise_vector(batch_size, self.noise_n)))
# train D
fake_x = self.G(noise_x)
real_out = self.D(real_x)
fake_out = self.D(fake_x.detach())
D_real = -torch.mean(real_out)
D_fake = torch.mean(fake_out)
D_loss = D_real + D_fake
self.reset_grad()
D_loss.backward()
self.D_optimizer.step()
# Log
loss = {}
loss['D/loss_real'] = D_real.data[0]
loss['D/loss_fake'] = D_fake.data[0]
loss['D/loss'] = D_loss.data[0]
# choose one in below two
# Clip weights of D
# for p in self.D.parameters():
# p.data.clamp_(-self.clip_value, clip_value)
# Gradients penalty, WGAP-GP
alpha = torch.rand(real_x.size(0), 1, 1,
1).cuda().expand_as(real_x)
# print(alpha.shape, real_x.shape, fake_x.shape)
interpolated = Variable(alpha * real_x.data +
(1 - alpha) * fake_x.data,
requires_grad=True)
gp_out = self.D(interpolated)
grad = torch.autograd.grad(outputs=gp_out,
inputs=interpolated,
grad_outputs=torch.ones(
gp_out.size()).cuda(),
retain_graph=True,
create_graph=True,
only_inputs=True)[0]
grad = grad.view(grad.size(0), -1)
grad_l2norm = torch.sqrt(torch.sum(grad**2, dim=1))
d_loss_gp = torch.mean((grad_l2norm - 1)**2)
# Backward + Optimize
d_loss = self.lambda_gp * d_loss_gp
self.reset_grad()
d_loss.backward()
self.D_optimizer.step()
# Train G
if (i + 1) % self.D_train_step == 0:
fake_out = self.D(self.G(noise_x))
G_loss = -torch.mean(fake_out)
self.reset_grad()
G_loss.backward()
self.G_optimizer.step()
loss['G/loss'] = G_loss.data[0]
# Print log
if (i + 1) % self.log_step == 0:
log = "Epoch: {}/{}, Iter: {}/{}".format(
e + 1, self.epoch, i + 1, len(self.data_loader))
for tag, value in loss.items():
log += ", {}: {:.4f}".format(tag, value)
print(log)
if self.use_tensorboard:
for tag, value in loss.items():
self.logger.scalar_summary(
tag, value,
e * len(self.data_loader) + i + 1)
# Save images
if (e + 1) % self.save_image_step == 0:
noise_x = self.to_var(
torch.FloatTensor(noise_vector(16, self.noise_n)))
fake_image = self.G(noise_x)
save_image(
self.denorm(fake_image.data),
os.path.join(self.image_save_path,
"{}_fake.png".format(e + 1)))
if (e + 1) % self.model_save_step == 0:
torch.save(
self.G.state_dict(),
os.path.join(self.model_save_path,
"{}_G.pth".format(e + 1)))
torch.save(
self.D.state_dict(),
os.path.join(self.model_save_path,
"{}_D.pth".format(e + 1)))
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import time
from selenium.common.exceptions import NoSuchElementException
import os
from commons.logger import Logger
log = Logger(logger='basepage').get_log()
class BasePage(object):
def __init__(self, driver):
self.driver = driver
def quit_browser(self):
self.driver.quit()
def forward(self):
self.driver.forward()
def back(self):
self.driver.back()
def wait(self, seconds):
self.driver.implicity_wait(seconds)
log.info('implicity wait %s s' % seconds)
def close(self):
try:
self.driver.close()
#!/usr/bin/env python3
# -*- coding:utf-8 -*-
import configparser
import os
from selenium import webdriver
from commons.logger import Logger
from commons.get_config import get_config
log = Logger(logger='browser_engine').get_log()
class BrowserEngine(object):
# 浏览器驱动文件
tool_dir = os.path.split(os.path.dirname(__file__))[0] + '/tools/'
chrome_driver = tool_dir + 'chromedriver.exe'
ie_driver = tool_dir + 'IEDriverServer.exe'
def __init__(self, driver):
self.driver = driver
def open_browser(self, driver):
browser = get_config('browser', 'browser')
log.info('browser_type: %s' % browser)
url = get_config('test_server', 'url')
log.info('Test server: %s' % url)
if 'ie' in browser.lower():
self.driver = webdriver.Ie(self.ie_driver)
log.info('Starting ie browser')
class Solver(object):
def __init__(self, data_loader, config):
self.data_loader = data_loader
self.noise_n = config.noise_n
self.G_last_act = last_act(config.G_last_act)
self.D_out_n = config.D_out_n
self.D_last_act = last_act(config.D_last_act)
self.G_lr = config.G_lr
self.D_lr = config.D_lr
self.beta1 = config.beta1
self.beta2 = config.beta2
self.epoch = config.epoch
self.batch_size = config.batch_size
self.D_train_step = config.D_train_step
self.save_image_step = config.save_image_step
self.log_step = config.log_step
self.model_save_step = config.model_save_step
self.gamma = config.gamma
self.lambda_k = config.lambda_k
self.model_save_path = config.model_save_path
self.log_save_path = config.log_save_path
self.image_save_path = config.image_save_path
self.use_tensorboard = config.use_tensorboard
self.pretrained_model = config.pretrained_model
self.build_model()
if self.use_tensorboard is not None:
self.build_tensorboard()
if self.pretrained_model is not None:
if len(self.pretrained_model) != 2:
raise "must have both G and D pretrained parameters, and G is first, D is second"
self.load_pretrained_model()
def build_model(self):
self.G = Generator(self.noise_n, self.G_last_act)
self.D = BEGAN_Discriminator(self.noise_n // 2, self.D_last_act)
self.G_optimizer = torch.optim.Adam(self.G.parameters(), self.G_lr,
[self.beta1, self.beta2])
self.D_optimizer = torch.optim.Adam(self.D.parameters(), self.D_lr,
[self.beta1, self.beta2])
if torch.cuda.is_available():
self.G.cuda()
self.D.cuda()
def build_tensorboard(self):
from commons.logger import Logger
self.logger = Logger(self.log_save_path)
def load_pretrained_model(self):
self.G.load_state_dict(torch.load(self.pretrained_model[0]))
self.D.load_state_dict(torch.load(self.pretrained_model[1]))
def denorm(self, x):
out = (x + 1) / 2
return out.clamp_(0, 1)
def reset_grad(self):
self.G_optimizer.zero_grad()
self.D_optimizer.zero_grad()
def to_var(self, x, volatile=False):
if torch.cuda.is_available():
x = x.cuda()
return Variable(x, volatile=volatile)
def train(self):
print(len(self.data_loader))
k = 0.0
for e in range(self.epoch):
for i, batch_images in enumerate(self.data_loader):
batch_size = batch_images.size(0)
real_x = self.to_var(batch_images)
noise_x = self.to_var(
torch.FloatTensor(noise_vector(batch_size, self.noise_n)))
# train D
fake_x = self.G(noise_x)
real_out = self.D(real_x)
fake_out = self.D(fake_x.detach())
# print(real_out.shape, real_x.shape, fake_x.shape, fake_out.shape)
D_real = torch.mean(torch.abs(real_out - real_x))
D_fake = torch.mean(torch.abs(fake_out - fake_x))
D_loss = D_real - k * D_fake
self.reset_grad()
D_loss.backward()
self.D_optimizer.step()
# Log
loss = {}
loss['D/loss_real'] = D_real.data[0]
loss['D/loss_fake'] = D_fake.data[0]
loss['D/loss'] = D_loss.data[0]
# Train G
if (i + 1) % self.D_train_step == 0:
fake_x = self.G(noise_x)
fake_out = self.D(fake_x)
G_loss = torch.mean(torch.abs(fake_out - fake_x))
self.reset_grad()
G_loss.backward()
self.G_optimizer.step()
loss['G/loss'] = G_loss.data[0]
# Update K
balance = (self.gamma * D_real - D_fake).data[0]
k += self.lambda_k * balance
k = max(min(1, k), 0)
loss['K'] = k
loss['M'] = D_real.cpu().data[0] + np.abs(balance.cpu())
# Print log
if (i + 1) % self.log_step == 0:
log = "Epoch: {}/{}, Iter: {}/{}".format(
e + 1, self.epoch, i + 1, len(self.data_loader))
for tag, value in loss.items():
log += ", {}: {:.4f}".format(tag, value)
print(log)
if self.use_tensorboard:
for tag, value in loss.items():
self.logger.scalar_summary(
tag, value,
e * len(self.data_loader) + i + 1)
# Save images
if (e + 1) % self.save_image_step == 0:
noise_x = self.to_var(
torch.FloatTensor(noise_vector(32, self.noise_n)))
fake_image = self.G(noise_x)
save_image(
self.denorm(fake_image.data),
os.path.join(self.image_save_path,
"{}_fake.png".format(e + 1)))
if (e + 1) % self.model_save_step == 0:
torch.save(
self.G.state_dict(),
os.path.join(self.model_save_path,
"{}_G.pth".format(e + 1)))
torch.save(
self.D.state_dict(),
os.path.join(self.model_save_path,
"{}_D.pth".format(e + 1)))