def initialize(modules: list) -> None:
if "entrance" in modules:
parser = argparse.ArgumentParser()
parser.add_argument("-e",
"--env",
action="store",
dest="env",
help="enviroment of server. prod|test|dev")
parser.add_argument("-p",
"--port",
action="store",
type=int,
dest="port",
default=6003,
help="port of running iplive node")
if len(sys.argv) < 2:
parser.print_help()
sys.exit()
args = parser.parse_args()
if args.env not in ["dev", "test", "prod"]:
print("enviroment not support")
sys.exit()
env = enviroment[args.env]
conf['env'] = env
conf['port'] = args.port
# if "logger" in modules:
# conf['logger'] = get_logger('iplives', on_screen=True, level=env['level'])
if "logger" in modules:
conf['logger'] = get_logger('iplives', on_screen=True, level="info")
# if "mysql" in modules:
# conf['mysql'] = MySqlOB(host=env['mysql']['host'], user=env['mysql']['user'],
# passwd=env['mysql']['passwd'], port=env['mysql']['port'],
# db=env['mysql']['db'])
if "mysql" in modules:
conf['mysql'] = MySqlOB()
if "redis" in modules:
conf['redis'] = Redispool()
def initialize():
parser = argparse.ArgumentParser()
parser.add_argument("-e",
"--env",
action="store",
dest="env",
help="enviroment of server. prod|test|dev")
# parser.add_argument("-p", "--port", action="store", dest="port", default=6003,
# help="port of running iplive node")
args = parser.parse_args()
if args.env not in ["dev", "prod"]:
print("enviroment not support")
sys.exit()
env = enviroment[args.env]
conf['env'] = env
if 'logger' not in conf:
conf['logger'] = get_logger("blog", on_screen=True, level=env['level'])
parse_articles()
from kits.utils import blfilter
from kits.utils import gget
from kits.redispool import Redispool
from config.conf import config
from kits.constants import RETOBJ
from kits.constants import ENGINE
from kits.constants import COOKIES
from kits.constants import HEADERS
from kits.constants import QUEUE_NAME
from kits.deal_domain import get_response
from kits.deal_domain import divide_article
from kits.my_exception import MyException
config['redis'] = Redispool(queue=QUEUE_NAME)
config['logger'] = get_logger("seoScrapy")
class SeoScrapy(object):
"""SEO data Scrapy"""
def __init__(self):
self.domain = ''
self.url = ''
self.pro_type = ''
self.res = ''
self.include = None
self.pre_size = ''
self.post_size = ''
self.gRate = ''
self.response = None
def initialize():
if 'redis' not in conf:
conf['redis'] = Redispool(),
if 'logger' not in conf:
conf['logger'] = get_logger('acceptRequest')
# encoding=utf8
"""Redis pool using one module in different files"""
import json
import redis
import time
import os
from functools import wraps
from kits.log import get_logger
ROOT_PATH = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
REDIS_LOGGER = get_logger('redis')
def redis_excepts(orig_func):
"""try excepts around each query"""
@wraps(orig_func)
def wrapper(*args, **kwargs):
try:
return orig_func(*args, **kwargs)
except redis.exceptions.ConnectionError as err:
REDIS_LOGGER.error(err)
except redis.exceptions.TimeoutError as err:
REDIS_LOGGER.error(err)
except Exception:
REDIS_LOGGER.critical("Exception", exc_info=True)
raise Exception("Error on redis. Check the log file")
return wrapper
def get_connection():