def initialize(modules: list) -> None: if "entrance" in modules: parser = argparse.ArgumentParser() parser.add_argument("-e", "--env", action="store", dest="env", help="enviroment of server. prod|test|dev") parser.add_argument("-p", "--port", action="store", type=int, dest="port", default=6003, help="port of running iplive node") if len(sys.argv) < 2: parser.print_help() sys.exit() args = parser.parse_args() if args.env not in ["dev", "test", "prod"]: print("enviroment not support") sys.exit() env = enviroment[args.env] conf['env'] = env conf['port'] = args.port # if "logger" in modules: # conf['logger'] = get_logger('iplives', on_screen=True, level=env['level']) if "logger" in modules: conf['logger'] = get_logger('iplives', on_screen=True, level="info") # if "mysql" in modules: # conf['mysql'] = MySqlOB(host=env['mysql']['host'], user=env['mysql']['user'], # passwd=env['mysql']['passwd'], port=env['mysql']['port'], # db=env['mysql']['db']) if "mysql" in modules: conf['mysql'] = MySqlOB() if "redis" in modules: conf['redis'] = Redispool()
def initialize(): parser = argparse.ArgumentParser() parser.add_argument("-e", "--env", action="store", dest="env", help="enviroment of server. prod|test|dev") # parser.add_argument("-p", "--port", action="store", dest="port", default=6003, # help="port of running iplive node") args = parser.parse_args() if args.env not in ["dev", "prod"]: print("enviroment not support") sys.exit() env = enviroment[args.env] conf['env'] = env if 'logger' not in conf: conf['logger'] = get_logger("blog", on_screen=True, level=env['level']) parse_articles()
from kits.utils import blfilter from kits.utils import gget from kits.redispool import Redispool from config.conf import config from kits.constants import RETOBJ from kits.constants import ENGINE from kits.constants import COOKIES from kits.constants import HEADERS from kits.constants import QUEUE_NAME from kits.deal_domain import get_response from kits.deal_domain import divide_article from kits.my_exception import MyException config['redis'] = Redispool(queue=QUEUE_NAME) config['logger'] = get_logger("seoScrapy") class SeoScrapy(object): """SEO data Scrapy""" def __init__(self): self.domain = '' self.url = '' self.pro_type = '' self.res = '' self.include = None self.pre_size = '' self.post_size = '' self.gRate = '' self.response = None
def initialize(): if 'redis' not in conf: conf['redis'] = Redispool(), if 'logger' not in conf: conf['logger'] = get_logger('acceptRequest')
# encoding=utf8 """Redis pool using one module in different files""" import json import redis import time import os from functools import wraps from kits.log import get_logger ROOT_PATH = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) REDIS_LOGGER = get_logger('redis') def redis_excepts(orig_func): """try excepts around each query""" @wraps(orig_func) def wrapper(*args, **kwargs): try: return orig_func(*args, **kwargs) except redis.exceptions.ConnectionError as err: REDIS_LOGGER.error(err) except redis.exceptions.TimeoutError as err: REDIS_LOGGER.error(err) except Exception: REDIS_LOGGER.critical("Exception", exc_info=True) raise Exception("Error on redis. Check the log file") return wrapper def get_connection():