def setUpClass(cls) -> None:
cfg = Config()
cls.username = cfg.get("env").get("Username")
cls.entcode = cfg.get("env").get("EntCode")
cls.passwd = cfg.get("env").get("Passwd")
driver = BrowserDriver(cls)
cls.driver = driver.open_browser(cls)
cls.driver.find_element_by_xpath("//div/input[@name='username']").send_keys(cls.username)
cls.driver.find_element_by_xpath("//div/input[@name='entcode']").send_keys(cls.entcode)
cls.driver.find_element_by_xpath("//div/input[@name='password']").send_keys(cls.passwd)
cls.driver.find_element_by_xpath("//div[contains(text(),'登录')]").click()
def __init__(self, logger_name='Autotest'):
self.logger = logging.getLogger(logger_name)
logging.root.setLevel(logging.NOTSET)
self.log_file_name = Config().get('log').get('file_name')
self.backup_count = Config().get('log').get('backup')
# 日志输出级别
self.console_output_level = Config().get('log').get('console_level')
self.file_output_level = Config().get('log').get('file_level')
# 日志输出格式
self.formatter = logging.Formatter(Config().get('log').get('pattern'))
def __init__(self):
self.title = Config().get('mail').get('title')
self.message = Config().get('mail').get('message')
self.files = Config().get('mail').get('path')
self.msg = MIMEMultipart('related')
self.server = Config().get('mail').get('server')
self.sender = Config().get('mail').get('sender')
self.receiver = Config().get('mail').get('receiver')
self.password = Config().get('mail').get('password')
class BrowserDriver(object):
path = DRIVER_PATH
def __init__(self, driver):
self.driver = driver
self.config = Config()
def open_browser(self, driver):
browser = self.config.get("browser").get("BrowserName")
logger.info("浏览器:%s" % browser)
url = self.config.get("env").get("URL")
logger.info("访问:%s" % url)
if browser == "Chrome":
chrome_options = Options()
chrome_options.add_argument(
'--no-sandbox') # 解决DevToolsActivePort文件不存在的报错
chrome_options.add_argument('--start-maximized') # 指定浏览器分辨率
chrome_options.add_argument(
'--disable-gpu') # 谷歌文档提到需要加上这个属性来规避bug
chrome_options.add_argument('--hide-scrollbars') # 隐藏滚动条, 应对一些特殊页面
# chrome_options.add_argument('blink-settings=imagesEnabled=false') # 不加载图片, 提升速度
# chrome_options.add_argument('--headless') # 浏览器不提供可视化页面. linux下如果系统不支持可视化不加这条会启动失败
chrome_options.add_argument('--disable-extensions')
chrome_options.add_argument('lang=zh_CN.UTF-8')
driver = webdriver.Chrome(executable_path=os.path.join(
DRIVER_PATH, "chromedriver.exe"),
options=chrome_options)
logger.info("启动谷歌浏览器")
elif browser == "IE":
driver = webdriver.Ie(os.path.join(DRIVER_PATH, "xxxx.exe"))
logger.info("启动IE浏览器")
elif browser == "Firefox":
driver = webdriver.Firefox(os.path.join(DRIVER_PATH, "xxxx.exe"))
logger.info("启动FireFox浏览器")
driver.get(url)
logger.info("打开URL: %s" % url)
driver.maximize_window()
logger.info("全屏当前窗口")
driver.implicitly_wait(5)
logger.info("设置5秒隐式等待时间")
return driver
def quit_browser(self):
logger.info("关闭浏览器")
self.driver.quit()
def __init__(self):
self.config = Config()
self.clock = pygame.time.Clock()
# info_object = pygame.display.Info()
# self.size = self.width, self.height = info_object.current_w, info_object.current_h
self.width = int(1800)
self.height = int(1000)
self.size = self.width, self.height
self.screen = pygame.display.set_mode((1800, 1000))
self.current_screen = None
self.screens = dict() # {INTRO_SCREEN: IntroScreenController(self), }
self.set_current_screen(self.config.values[CONFIG_FIRST_SCREEN])
self.game = None
class EmailSet:
def __init__(self):
self.title = Config().get('mail').get('title')
self.message = Config().get('mail').get('message')
self.files = Config().get('mail').get('path')
self.msg = MIMEMultipart('related')
self.server = Config().get('mail').get('server')
self.sender = Config().get('mail').get('sender')
self.receiver = Config().get('mail').get('receiver')
self.password = Config().get('mail').get('password')
def attach_file(self, att_file):
att = MIMEText(open(att_file, 'rb').read(), 'bsae64', 'utf-8')
att["Content-Type"] = 'application/octet-stream'
file_name = re.split(r'[\\|/]', att_file)
filename = file_name[-1]
att["Content-Disposition"] = 'attachment; filename="%s"' % filename
self.msg.attach(att)
def send(self):
self.msg['Subject'] = self.title
self.msg['From'] = self.sender
self.msg['To'] = self.receiver
# 邮件正文
if self.message:
self.msg.attach(MIMEText(self.message, "html", "utf-8"))
try:
smtp_server = smtplib.SMTP_SSL(self.server,465)
except (gaierror and error) as e:
logger.exception('发送邮件失败,无法连接到SMTP服务器，检查网络以及SMTP服务器. %s', e)
else:
try:
smtp_server.login(self.sender, self.password)
except smtplib.SMTPAuthenticationError as e:
logger.exception('用户名密码验证失败！%s', e)
else:
smtp_server.sendmail(self.sender, self.receiver.split(','), self.msg.as_string())
finally:
smtp_server.quit()
logger.info('发送邮件"{0}"成功! 收件人：{1}。如果没有收到邮件，请检查垃圾箱，'
'同时检查收件人地址是否正确'.format(self.title, self.receiver))
from src.aws import client as aws
from src.common.config import Config
from src.model.todo import Todo
if __name__ == '__main__':
Config.setup()
table = aws.dynamo().create_table(**Todo.schema)
table.meta.client.get_waiter('table_exists').wait(TableName=Todo.table())
print('############')
print('Dynamo Ready')
print('############')
@classmethod
def setUpClass(cls) -> None:
cfg = Config()
cls.username = cfg.get("env").get("Username")
cls.entcode = cfg.get("env").get("EntCode")
cls.passwd = cfg.get("env").get("Passwd")
driver = BrowserDriver(cls)
cls.driver = driver.open_browser(cls)
cls.driver.find_element_by_xpath("//div/input[@name='username']").send_keys(cls.username)
cls.driver.find_element_by_xpath("//div/input[@name='entcode']").send_keys(cls.entcode)
cls.driver.find_element_by_xpath("//div/input[@name='password']").send_keys(cls.passwd)
cls.driver.find_element_by_xpath("//div[contains(text(),'登录')]").click()
def setUp(self) -> None:
pass
def tearDown(self) -> None:
pass
@classmethod
def tearDownClass(cls) -> None:
cls.driver.quit()
if __name__ == "__main__":
print(Config().get("env").get("URL"))
def main(file_name):
# Load json
with open(file_name, "r") as f:
setting_json = json.load(f)
config = Config(setting_json)
config_log = config.get_log()
print(config_log)
os.makedirs(config.trainer_config["path"], exist_ok=True)
with open(config.trainer_config["path"] + "/config.json", "w") as f:
json.dump(setting_json, f)
with open(config.trainer_config["path"] + "/config.log", "w") as f:
f.write(config_log)
# Setting seed
random.seed(config.seed)
torch.manual_seed(config.seed)
if config.device != "cpu":
torch.cuda.manual_seed(config.seed)
torch.cuda.manual_seed_all(config.seed)
torch.backends.cudnn.deterministic = True
# Define field
WORD = data.Field(
include_lengths=True,
batch_first=True,
lower=True,
preprocessing=data.Pipeline(lambda w: re.sub('\d', '0', w)
if config.is_digit else w))
CHAR_NESTING = data.Field(
tokenize=list,
batch_first=True,
lower=config.is_lower,
init_token=START_TAG,
eos_token=STOP_TAG,
preprocessing=data.Pipeline(lambda s: re.sub('\d', '0', s)
if config.is_digit else s))
CHAR = data.NestedField(CHAR_NESTING, include_lengths=True)
LABEL = data.Field(unk_token=UNLABELED_TAG, batch_first=True)
fields = [(('word', 'char'), (WORD, CHAR)), ('label', LABEL)]
# Load datasets
train_dataset, valid_dataset, test_dataset = Conll2003Dataset.splits(
fields=fields,
path=config.dataset_path,
separator=" ",
train="eng.train",
validation="eng.testa",
test="eng.testb")
# Build vocab
WORD.build_vocab(train_dataset,
valid_dataset,
test_dataset,
vectors=GloVe(name='6B', dim='100'))
CHAR.build_vocab(train_dataset, valid_dataset, test_dataset)
LABEL.build_vocab(train_dataset, valid_dataset, test_dataset)
# UNKNOWN tag is -1
LABEL.vocab.stoi = Counter({k: v - 1 for k, v in LABEL.vocab.stoi.items()})
# Don't count UNKNOWN tag
num_tags = len(LABEL.vocab) - 1
assert LABEL.vocab.stoi[UNLABELED_TAG] == UNLABELED_ID
if config.inference_type in ["Simple", "Hard"]:
LABEL.vocab.stoi[UNLABELED_TAG] = LABEL.vocab.stoi["O"]
# Create trainer
if config.inference_type == "Hard":
trainer = HardTrainer(num_tags,
LABEL.vocab,
CHAR.vocab,
WORD.vocab,
config.emb_dict,
config,
config.trainer_config,
train_dataset,
valid_dataset,
test_dataset=test_dataset,
label_dict=LABEL.vocab.stoi,
is_every_all_train=config.is_every_all_train)
else:
model = BiLSTM_CRF(num_tags,
LABEL.vocab,
CHAR.vocab,
WORD.vocab,
config.emb_dict,
dropout_rate=config.dropout_rate,
inference_type=config.inference_type)
if config.device != "cpu":
model = model.to(config.device)
trainer = Trainer(model,
config.trainer_config,
train_dataset,
valid_dataset,
test_dataset=test_dataset,
label_dict=LABEL.vocab.stoi)
trainer.train(config.trainer_config["epochs"])
def main(file_name):
with open(file_name, "r") as f:
setting_json = json.load(f)
config = Config(setting_json)
os.makedirs(config.SAVE_MODEL_PATH, exist_ok=True)
with open(config.SAVE_MODEL_PATH + "/config.json", "w") as f:
json.dump(setting_json, f)
random.seed(config.SEED)
np.random.seed(config.SEED)
torch.manual_seed(config.SEED)
if config.device == "GPU":
torch.cuda.manual_seed(config.SEED)
torch.cuda.manual_seed_all(config.SEED)
dataset = Conll2003Dataset(config.BATCH_SIZE,
config.DATASET_PATH,
word_emb_dim=config.WORD_EMBEDDING_DIM,
pretrain_type=config.WORD_EMBEDDING_TYPE,
unlabel_to_other=config.unlabel_to_other,
device=config.device)
num_tags = dataset.num_tags
label2idx, idx2labels = dataset.label2idx, dataset.idx2labels
char2idx, idx2char = dataset.char2idx, dataset.idx2char
word_embedding = dataset.word_embedding
if config.MODEL_TYPE == "crf" or config.MODEL_TYPE == "simple":
model = BiLSTM_CRF(num_tags,
label2idx,
idx2labels,
word_embedding,
config.WORD_EMBEDDING_DIM,
config.HIDDEN_DIM,
config.CHAR_EMBEDDING_DIM,
config.CHAR_HIDDEN_DIM,
char2idx,
idx2char,
dropout_rate=config.DROPOUT_RATE,
device=config.device)
elif config.MODEL_TYPE == "fuzzy_crf":
model = BiLSTM_CRF(num_tags,
label2idx,
idx2labels,
word_embedding,
config.WORD_EMBEDDING_DIM,
config.HIDDEN_DIM,
config.CHAR_EMBEDDING_DIM,
config.CHAR_HIDDEN_DIM,
char2idx,
idx2char,
dropout_rate=config.DROPOUT_RATE,
inference="FuzzyCRF",
device=config.device)
elif config.MODEL_TYPE == "hard_crf":
model = BiLSTM_Hard_CRF(num_tags,
label2idx,
idx2labels,
word_embedding,
config.WORD_EMBEDDING_DIM,
config.HIDDEN_DIM,
config.CHAR_EMBEDDING_DIM,
config.CHAR_HIDDEN_DIM,
char2idx,
idx2char,
dropout_rate=config.DROPOUT_RATE,
device=config.device)
if config.device != "cpu":
model = model.cuda(config.device)
train_iter = dataset.train_batch
valid_iter = dataset.valid_batch
test_iter = dataset.test_batch
if config.MODEL_TYPE == "hard_crf":
SUB_EPOCHS = setting_json["train"]["sub_num_epochs"]
trainer = HardTrainer(model,
train_iter,
valid_iter=valid_iter,
test_iter=test_iter,
sub_num_epochs=SUB_EPOCHS,
label_dict=label2idx,
learning_rate=config.LEAENING_RATE,
clipping=config.CLIPPING,
save_path=config.SAVE_MODEL_PATH,
train_only=config.TRAIN_ONLY,
force_save=True,
device=config.device)
else:
trainer = Trainer(model,
train_iter,
valid_iter=valid_iter,
test_iter=test_iter,
label_dict=label2idx,
learning_rate=config.LEAENING_RATE,
clipping=config.CLIPPING,
save_path=config.SAVE_MODEL_PATH,
train_only=config.TRAIN_ONLY,
force_save=True)
trainer.train(config.EPOCHS)
def test_config(self):
Config.setup()
self.assertTrue(Config.version != 'unknown')
def __init__(self, driver):
self.driver = driver
self.config = Config()