def __init__(self, logger_name='Autotest'):
self.logger = logging.getLogger(logger_name)
logging.root.setLevel(logging.NOTSET)
self.log_file_name = Config().get('log').get('file_name')
self.backup_count = Config().get('log').get('backup')
# 日志输出级别
self.console_output_level = Config().get('log').get('console_level')
self.file_output_level = Config().get('log').get('file_level')
# 日志输出格式
self.formatter = logging.Formatter(Config().get('log').get('pattern'))
def __init__(self):
self.title = Config().get('mail').get('title')
self.message = Config().get('mail').get('message')
self.files = Config().get('mail').get('path')
self.msg = MIMEMultipart('related')
self.server = Config().get('mail').get('server')
self.sender = Config().get('mail').get('sender')
self.receiver = Config().get('mail').get('receiver')
self.password = Config().get('mail').get('password')
def setUpClass(cls) -> None:
cfg = Config()
cls.username = cfg.get("env").get("Username")
cls.entcode = cfg.get("env").get("EntCode")
cls.passwd = cfg.get("env").get("Passwd")
driver = BrowserDriver(cls)
cls.driver = driver.open_browser(cls)
cls.driver.find_element_by_xpath("//div/input[@name='username']").send_keys(cls.username)
cls.driver.find_element_by_xpath("//div/input[@name='entcode']").send_keys(cls.entcode)
cls.driver.find_element_by_xpath("//div/input[@name='password']").send_keys(cls.passwd)
cls.driver.find_element_by_xpath("//div[contains(text(),'登录')]").click()
def __init__(self):
self.config = Config()
self.clock = pygame.time.Clock()
# info_object = pygame.display.Info()
# self.size = self.width, self.height = info_object.current_w, info_object.current_h
self.width = int(1800)
self.height = int(1000)
self.size = self.width, self.height
self.screen = pygame.display.set_mode((1800, 1000))
self.current_screen = None
self.screens = dict() # {INTRO_SCREEN: IntroScreenController(self), }
self.set_current_screen(self.config.values[CONFIG_FIRST_SCREEN])
self.game = None
@classmethod
def setUpClass(cls) -> None:
cfg = Config()
cls.username = cfg.get("env").get("Username")
cls.entcode = cfg.get("env").get("EntCode")
cls.passwd = cfg.get("env").get("Passwd")
driver = BrowserDriver(cls)
cls.driver = driver.open_browser(cls)
cls.driver.find_element_by_xpath("//div/input[@name='username']").send_keys(cls.username)
cls.driver.find_element_by_xpath("//div/input[@name='entcode']").send_keys(cls.entcode)
cls.driver.find_element_by_xpath("//div/input[@name='password']").send_keys(cls.passwd)
cls.driver.find_element_by_xpath("//div[contains(text(),'登录')]").click()
def setUp(self) -> None:
pass
def tearDown(self) -> None:
pass
@classmethod
def tearDownClass(cls) -> None:
cls.driver.quit()
if __name__ == "__main__":
print(Config().get("env").get("URL"))
def main(file_name):
# Load json
with open(file_name, "r") as f:
setting_json = json.load(f)
config = Config(setting_json)
config_log = config.get_log()
print(config_log)
os.makedirs(config.trainer_config["path"], exist_ok=True)
with open(config.trainer_config["path"] + "/config.json", "w") as f:
json.dump(setting_json, f)
with open(config.trainer_config["path"] + "/config.log", "w") as f:
f.write(config_log)
# Setting seed
random.seed(config.seed)
torch.manual_seed(config.seed)
if config.device != "cpu":
torch.cuda.manual_seed(config.seed)
torch.cuda.manual_seed_all(config.seed)
torch.backends.cudnn.deterministic = True
# Define field
WORD = data.Field(
include_lengths=True,
batch_first=True,
lower=True,
preprocessing=data.Pipeline(lambda w: re.sub('\d', '0', w)
if config.is_digit else w))
CHAR_NESTING = data.Field(
tokenize=list,
batch_first=True,
lower=config.is_lower,
init_token=START_TAG,
eos_token=STOP_TAG,
preprocessing=data.Pipeline(lambda s: re.sub('\d', '0', s)
if config.is_digit else s))
CHAR = data.NestedField(CHAR_NESTING, include_lengths=True)
LABEL = data.Field(unk_token=UNLABELED_TAG, batch_first=True)
fields = [(('word', 'char'), (WORD, CHAR)), ('label', LABEL)]
# Load datasets
train_dataset, valid_dataset, test_dataset = Conll2003Dataset.splits(
fields=fields,
path=config.dataset_path,
separator=" ",
train="eng.train",
validation="eng.testa",
test="eng.testb")
# Build vocab
WORD.build_vocab(train_dataset,
valid_dataset,
test_dataset,
vectors=GloVe(name='6B', dim='100'))
CHAR.build_vocab(train_dataset, valid_dataset, test_dataset)
LABEL.build_vocab(train_dataset, valid_dataset, test_dataset)
# UNKNOWN tag is -1
LABEL.vocab.stoi = Counter({k: v - 1 for k, v in LABEL.vocab.stoi.items()})
# Don't count UNKNOWN tag
num_tags = len(LABEL.vocab) - 1
assert LABEL.vocab.stoi[UNLABELED_TAG] == UNLABELED_ID
if config.inference_type in ["Simple", "Hard"]:
LABEL.vocab.stoi[UNLABELED_TAG] = LABEL.vocab.stoi["O"]
# Create trainer
if config.inference_type == "Hard":
trainer = HardTrainer(num_tags,
LABEL.vocab,
CHAR.vocab,
WORD.vocab,
config.emb_dict,
config,
config.trainer_config,
train_dataset,
valid_dataset,
test_dataset=test_dataset,
label_dict=LABEL.vocab.stoi,
is_every_all_train=config.is_every_all_train)
else:
model = BiLSTM_CRF(num_tags,
LABEL.vocab,
CHAR.vocab,
WORD.vocab,
config.emb_dict,
dropout_rate=config.dropout_rate,
inference_type=config.inference_type)
if config.device != "cpu":
model = model.to(config.device)
trainer = Trainer(model,
config.trainer_config,
train_dataset,
valid_dataset,
test_dataset=test_dataset,
label_dict=LABEL.vocab.stoi)
trainer.train(config.trainer_config["epochs"])
def main(file_name):
with open(file_name, "r") as f:
setting_json = json.load(f)
config = Config(setting_json)
os.makedirs(config.SAVE_MODEL_PATH, exist_ok=True)
with open(config.SAVE_MODEL_PATH + "/config.json", "w") as f:
json.dump(setting_json, f)
random.seed(config.SEED)
np.random.seed(config.SEED)
torch.manual_seed(config.SEED)
if config.device == "GPU":
torch.cuda.manual_seed(config.SEED)
torch.cuda.manual_seed_all(config.SEED)
dataset = Conll2003Dataset(config.BATCH_SIZE,
config.DATASET_PATH,
word_emb_dim=config.WORD_EMBEDDING_DIM,
pretrain_type=config.WORD_EMBEDDING_TYPE,
unlabel_to_other=config.unlabel_to_other,
device=config.device)
num_tags = dataset.num_tags
label2idx, idx2labels = dataset.label2idx, dataset.idx2labels
char2idx, idx2char = dataset.char2idx, dataset.idx2char
word_embedding = dataset.word_embedding
if config.MODEL_TYPE == "crf" or config.MODEL_TYPE == "simple":
model = BiLSTM_CRF(num_tags,
label2idx,
idx2labels,
word_embedding,
config.WORD_EMBEDDING_DIM,
config.HIDDEN_DIM,
config.CHAR_EMBEDDING_DIM,
config.CHAR_HIDDEN_DIM,
char2idx,
idx2char,
dropout_rate=config.DROPOUT_RATE,
device=config.device)
elif config.MODEL_TYPE == "fuzzy_crf":
model = BiLSTM_CRF(num_tags,
label2idx,
idx2labels,
word_embedding,
config.WORD_EMBEDDING_DIM,
config.HIDDEN_DIM,
config.CHAR_EMBEDDING_DIM,
config.CHAR_HIDDEN_DIM,
char2idx,
idx2char,
dropout_rate=config.DROPOUT_RATE,
inference="FuzzyCRF",
device=config.device)
elif config.MODEL_TYPE == "hard_crf":
model = BiLSTM_Hard_CRF(num_tags,
label2idx,
idx2labels,
word_embedding,
config.WORD_EMBEDDING_DIM,
config.HIDDEN_DIM,
config.CHAR_EMBEDDING_DIM,
config.CHAR_HIDDEN_DIM,
char2idx,
idx2char,
dropout_rate=config.DROPOUT_RATE,
device=config.device)
if config.device != "cpu":
model = model.cuda(config.device)
train_iter = dataset.train_batch
valid_iter = dataset.valid_batch
test_iter = dataset.test_batch
if config.MODEL_TYPE == "hard_crf":
SUB_EPOCHS = setting_json["train"]["sub_num_epochs"]
trainer = HardTrainer(model,
train_iter,
valid_iter=valid_iter,
test_iter=test_iter,
sub_num_epochs=SUB_EPOCHS,
label_dict=label2idx,
learning_rate=config.LEAENING_RATE,
clipping=config.CLIPPING,
save_path=config.SAVE_MODEL_PATH,
train_only=config.TRAIN_ONLY,
force_save=True,
device=config.device)
else:
trainer = Trainer(model,
train_iter,
valid_iter=valid_iter,
test_iter=test_iter,
label_dict=label2idx,
learning_rate=config.LEAENING_RATE,
clipping=config.CLIPPING,
save_path=config.SAVE_MODEL_PATH,
train_only=config.TRAIN_ONLY,
force_save=True)
trainer.train(config.EPOCHS)
def __init__(self, driver):
self.driver = driver
self.config = Config()