def main():
# Calling data process activity
job_name = config["data_process_job"]
logger = c_utils.get_logger(job_name)
msg = ">> ========== Data Processing Job - {} started ... : {}".format(job_name, datetime.now())
print(msg)
logger.info(msg)
# Calling the raw data file processing function
status = data_process(logger)
if status == 'Success':
status = load_to_database()
msg = "\nThe Final Status of the job - {} execution is ... : {}".format(job_name, status)
print(msg)
logger.info(msg)
msg = ">> ========== Data Processing Job - {} finished ... : {}".format(job_name, datetime.now())
print(msg)
logger.info(msg)
from __future__ import print_function, division
import tensorflow as tf
from tensorflow.python.ops import rnn_cell
from tensorflow.python.ops import seq2seq
from common_utils import get_logger
import sys
import numpy
import pprint
MODEL_SCOPE = "char_rnn"
DEVICE_SCOPE = "/gpu:0"
logger = get_logger()
class Model(object):
def __init__(self, config):
"""Init model from provided configuration
Args:
config (dict): Model's configuration
Should have:
rnn_size: size of RNN hidden state
num_layers: number of RNN layers
rnn_type: lstm, rnn, or gru
batch_size: batch size
seq_length: sequence length
grad_clip: Clip gradient value by this value
vocab_size: size of vocabulary
infer: True/False, if True, use the predicted output
to feed back to RNN insted of gold target output.
is_train: True if is training
from pathlib import Path
import sys
sys.path.append('..')
from load import load, FILE_NAME_GENRE_TRANSLATE
from common_utils import get_logger
FILE_NAME_MERGE_GENRE_TRANSLATE = str(
Path(__file__).parent.resolve() / 'data' / 'merge_genre_translate.json')
DIR = Path(__file__).parent.resolve()
FILE_NAME_BACKUP = DIR / 'backup'
FILE_NAME_BACKUP.mkdir(parents=True, exist_ok=True)
log = get_logger('merge_genre_translate.txt')
log.info('Start.')
backup_file_name = (FILE_NAME_BACKUP /
(DT.datetime.today().strftime('%d%m%y-%H%M%S_') +
Path(FILE_NAME_GENRE_TRANSLATE).name))
log.info(f'Save backup to: {backup_file_name}')
shutil.copy(FILE_NAME_GENRE_TRANSLATE, backup_file_name)
log.info('Load genres')
genre_translate = load()
log.info(
f'Current genres: {len(genre_translate)}. '
f'Null genres: {sum(1 for v in genre_translate.values() if v is None)}')
from src_file import SrcFiles
from qa_conclude_info import ans_libs, emoji_lib
# from qa_conclude_info import qa_host_info,qa_chat_info,self_label
# from qa_conclude_info import host_info, chatbot_info, user_info
import threading
from mpe_parse import MPEparse, jieba_add_word, mpe_add_rule
from nostanford_rule import NoStanfordRule
mpeParse = MPEparse()
jieba_add_word()
# mpe_add_rule(mpeParse)
cur_dir = os.getcwd()
if cur_dir.endswith("src"):
cur_dir = os.path.dirname(cur_dir)
logger = get_logger(os.path.join(cur_dir, "log", "match_for_api.log"))
logger.info("current dir:%s" % cur_dir)
if cur_dir.endswith("chatbot"):
cur_dir = os.path.join(cur_dir, "src")
app = Flask(__name__, template_folder=os.path.join(cur_dir, "template"))
app.secret_key = '\xf1\x92Y\xdf\x8ejY\x04\x96\xb4V\x88\xfb\xfc\xb5\x18F\xa3\xee\xb9\xb9t\x01\xf0\x96' #配置secret_key,否则不能实现session对话
bootstrap = Bootstrap(app) #Flask扩展一般都在创建实例时初始化，这行代码是Flask-Bootstrap的初始化方法
srcFiles = SrcFiles("convers_corpus")
proc_trie, idf_trie, answer_indexs = Match.load_file(srcFiles.store_path)
proc_ratio = Match.proc_ratio(0.2, 0.5)
src_trie = srcFiles.get_src_trie
g = Graph()
__author__ = 'ipetrash'
import datetime as DT
import json
from pathlib import Path
import shutil
import sys
sys.path.append('..')
sys.path.append('../genre_translate_file')
from db import Dump
from load import load
from common_utils import get_logger
log = get_logger('generate_games.txt')
DIR = Path(__file__).parent.resolve()
FILE_NAME_GAMES = str(DIR / 'game_by_genres.json')
FILE_NAME_BACKUP = DIR / 'backup'
FILE_NAME_BACKUP.mkdir(parents=True, exist_ok=True)
# Example: "Action", "Adventure" -> "Action-adventure"
GENRE_COMPRESSION = [
("Action", "Adventure", "Action-adventure"),
("Action", "RPG", "Action/RPG"),
("First-person", "Shooter", "FPS"),
("Survival", "Horror", "Survival horror"),
]
logger.info('----------------------------------')
logger.info('LM post epoch routine...')
done_training = run_post_epoch(train_ppl,
valid_ppl,
state,
opt,
sess=sess,
saver=saver,
best_prefix="best_lm",
latest_prefix="latest_lm")
logger.info('- Epoch time: {}s'.format(time.time() - epoch_time))
if done_training:
break
logger.info('Done training at epoch {}'.format(state.epoch + 1))
if __name__ == "__main__":
global_time = time.time()
parser = common_utils.get_common_argparse()
args = parser.parse_args()
opt = common_utils.Bunch.default_model_options()
opt.update_from_ns(args)
logger = common_utils.get_logger(opt.log_file_path)
if opt.debug:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
logger.info('Configurations:\n{}'.format(opt.__repr__()))
main(opt)
logger.info('Total time: {}s'.format(time.time() - global_time))
__author__ = 'ipetrash'
import json
import sys
sys.path.append('..')
from db import Dump
from load import FILE_NAME_GENRE_TRANSLATE, load
from common_utils import get_logger
from utils import send_sms
log = get_logger('genre_translate.txt')
def run():
log.info('Start load genres.')
genre_translate = load()
NEED_SMS = True
new_genres = []
is_first_run = not genre_translate
log.info(f'Current genres: {len(genre_translate)}')
for genre in Dump.get_all_genres():
if genre not in genre_translate:
@staticmethod
def get_game_genres(game_name):
if game_name == 'Foo':
raise Exception('Error')
return ['RGB-bar', 'Action-bar']
# Monkey Patch
def get_parsers():
return [FakeParser]
def get_games_list(): return ['Foo', 'Bar', 'Zet']
log = get_logger()
counter = AtomicCounter()
def run_parser(parser, games: list, max_num_request=5):
try:
pauses = [
('15 minutes', 15 * 60),
('30 minutes', 30 * 60),
('45 minutes', 45 * 60),
('1 hour', 60 * 60),
]
SITE_NAME = parser.get_site_name()
timeout = 3 # 3 seconds
MAX_TIMEOUT = 10 # 10 seconds
TIMEOUT_EVERY_N_GAMES = 50 # Every 50 games
