def main(): # Calling data process activity job_name = config["data_process_job"] logger = c_utils.get_logger(job_name) msg = ">> ========== Data Processing Job - {} started ... : {}".format(job_name, datetime.now()) print(msg) logger.info(msg) # Calling the raw data file processing function status = data_process(logger) if status == 'Success': status = load_to_database() msg = "\nThe Final Status of the job - {} execution is ... : {}".format(job_name, status) print(msg) logger.info(msg) msg = ">> ========== Data Processing Job - {} finished ... : {}".format(job_name, datetime.now()) print(msg) logger.info(msg)
from __future__ import print_function, division import tensorflow as tf from tensorflow.python.ops import rnn_cell from tensorflow.python.ops import seq2seq from common_utils import get_logger import sys import numpy import pprint MODEL_SCOPE = "char_rnn" DEVICE_SCOPE = "/gpu:0" logger = get_logger() class Model(object): def __init__(self, config): """Init model from provided configuration Args: config (dict): Model's configuration Should have: rnn_size: size of RNN hidden state num_layers: number of RNN layers rnn_type: lstm, rnn, or gru batch_size: batch size seq_length: sequence length grad_clip: Clip gradient value by this value vocab_size: size of vocabulary infer: True/False, if True, use the predicted output to feed back to RNN insted of gold target output. is_train: True if is training
from pathlib import Path import sys sys.path.append('..') from load import load, FILE_NAME_GENRE_TRANSLATE from common_utils import get_logger FILE_NAME_MERGE_GENRE_TRANSLATE = str( Path(__file__).parent.resolve() / 'data' / 'merge_genre_translate.json') DIR = Path(__file__).parent.resolve() FILE_NAME_BACKUP = DIR / 'backup' FILE_NAME_BACKUP.mkdir(parents=True, exist_ok=True) log = get_logger('merge_genre_translate.txt') log.info('Start.') backup_file_name = (FILE_NAME_BACKUP / (DT.datetime.today().strftime('%d%m%y-%H%M%S_') + Path(FILE_NAME_GENRE_TRANSLATE).name)) log.info(f'Save backup to: {backup_file_name}') shutil.copy(FILE_NAME_GENRE_TRANSLATE, backup_file_name) log.info('Load genres') genre_translate = load() log.info( f'Current genres: {len(genre_translate)}. ' f'Null genres: {sum(1 for v in genre_translate.values() if v is None)}')
from src_file import SrcFiles from qa_conclude_info import ans_libs, emoji_lib # from qa_conclude_info import qa_host_info,qa_chat_info,self_label # from qa_conclude_info import host_info, chatbot_info, user_info import threading from mpe_parse import MPEparse, jieba_add_word, mpe_add_rule from nostanford_rule import NoStanfordRule mpeParse = MPEparse() jieba_add_word() # mpe_add_rule(mpeParse) cur_dir = os.getcwd() if cur_dir.endswith("src"): cur_dir = os.path.dirname(cur_dir) logger = get_logger(os.path.join(cur_dir, "log", "match_for_api.log")) logger.info("current dir:%s" % cur_dir) if cur_dir.endswith("chatbot"): cur_dir = os.path.join(cur_dir, "src") app = Flask(__name__, template_folder=os.path.join(cur_dir, "template")) app.secret_key = '\xf1\x92Y\xdf\x8ejY\x04\x96\xb4V\x88\xfb\xfc\xb5\x18F\xa3\xee\xb9\xb9t\x01\xf0\x96' #配置secret_key,否则不能实现session对话 bootstrap = Bootstrap(app) #Flask扩展一般都在创建实例时初始化,这行代码是Flask-Bootstrap的初始化方法 srcFiles = SrcFiles("convers_corpus") proc_trie, idf_trie, answer_indexs = Match.load_file(srcFiles.store_path) proc_ratio = Match.proc_ratio(0.2, 0.5) src_trie = srcFiles.get_src_trie g = Graph()
__author__ = 'ipetrash' import datetime as DT import json from pathlib import Path import shutil import sys sys.path.append('..') sys.path.append('../genre_translate_file') from db import Dump from load import load from common_utils import get_logger log = get_logger('generate_games.txt') DIR = Path(__file__).parent.resolve() FILE_NAME_GAMES = str(DIR / 'game_by_genres.json') FILE_NAME_BACKUP = DIR / 'backup' FILE_NAME_BACKUP.mkdir(parents=True, exist_ok=True) # Example: "Action", "Adventure" -> "Action-adventure" GENRE_COMPRESSION = [ ("Action", "Adventure", "Action-adventure"), ("Action", "RPG", "Action/RPG"), ("First-person", "Shooter", "FPS"), ("Survival", "Horror", "Survival horror"), ]
logger.info('----------------------------------') logger.info('LM post epoch routine...') done_training = run_post_epoch(train_ppl, valid_ppl, state, opt, sess=sess, saver=saver, best_prefix="best_lm", latest_prefix="latest_lm") logger.info('- Epoch time: {}s'.format(time.time() - epoch_time)) if done_training: break logger.info('Done training at epoch {}'.format(state.epoch + 1)) if __name__ == "__main__": global_time = time.time() parser = common_utils.get_common_argparse() args = parser.parse_args() opt = common_utils.Bunch.default_model_options() opt.update_from_ns(args) logger = common_utils.get_logger(opt.log_file_path) if opt.debug: logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.INFO) logger.info('Configurations:\n{}'.format(opt.__repr__())) main(opt) logger.info('Total time: {}s'.format(time.time() - global_time))
__author__ = 'ipetrash' import json import sys sys.path.append('..') from db import Dump from load import FILE_NAME_GENRE_TRANSLATE, load from common_utils import get_logger from utils import send_sms log = get_logger('genre_translate.txt') def run(): log.info('Start load genres.') genre_translate = load() NEED_SMS = True new_genres = [] is_first_run = not genre_translate log.info(f'Current genres: {len(genre_translate)}') for genre in Dump.get_all_genres(): if genre not in genre_translate:
@staticmethod def get_game_genres(game_name): if game_name == 'Foo': raise Exception('Error') return ['RGB-bar', 'Action-bar'] # Monkey Patch def get_parsers(): return [FakeParser] def get_games_list(): return ['Foo', 'Bar', 'Zet'] log = get_logger() counter = AtomicCounter() def run_parser(parser, games: list, max_num_request=5): try: pauses = [ ('15 minutes', 15 * 60), ('30 minutes', 30 * 60), ('45 minutes', 45 * 60), ('1 hour', 60 * 60), ] SITE_NAME = parser.get_site_name() timeout = 3 # 3 seconds MAX_TIMEOUT = 10 # 10 seconds TIMEOUT_EVERY_N_GAMES = 50 # Every 50 games