def backup_count_table(t, flag):
'''备份统计表'''
count_b1_sql = count_backup_sql.format(tablename=house_list_name,
counttablename="%s_%s" %
(house_list_name, t),
columnname="run_date",
runtime=t)
count_b2_sql = count_backup_sql.format(tablename=price_trend_name,
counttablename="%s_%s" %
(price_trend_name, t[:-2]),
columnname="run_month",
runtime=t[:-2])
ps_mday = ConfigReader.read_section_key("quota", "ps_mday")
try:
db.execute(count_b1_sql)
db_optor_info("【%s】房源日常备份成功！" % t)
except Exception:
db_optor_info("【%s】房源日常备份失败！" % (t))
try:
if not flag:
db.execute(count_b2_sql)
db_optor_info("【-%s-】价格趋势备份成功！" % (t[:-2]))
except Exception:
db_optor_info("【-%s-】价格趋势备份失败！" % (t[:-2]))
class BaseServiceIntegration(object):
def __init__(self):
self.config = ConfigReader()
self.logger = logging.getLogger(__name__)
self.logger.debug("Loading config properties for AWS")
self.default_region = self.config.get_property(
self.config.SECTION_AWS_ACCOUNT, self.config.OPTION_DEFAULT_REGION)
def backup_table(db, t):
'''备份缓存表操作'''
b1_sql = rename_sql.format(fromname=house_list_name,
toname="%s_%s" % (house_list_name, t))
count_b1_sql = count_backup_sql.format(tablename=house_list_name,
counttablename=house_list_name,
columnname="run_date",
runtime=t)
db.execute(count_b1_sql)
db.execute(b1_sql)
ps_mday = ConfigReader.read_section_key("quota", "ps_mday")
if int(ps_mday) == int(time.localtime().tm_mday):
b2_sql = rename_sql.format(fromname=price_trend_name,
toname="%s_%s" % (price_trend_name, t[:-2]))
count_b2_sql = count_backup_sql.format(tablename=price_trend_name,
counttablename=price_trend_name,
columnname="run_month",
runtime=t[:-2])
db.execute(count_b2_sql)
db.execute(b2_sql)
create(db)
truncate_redis()
def __init__(self, section_name="redis_hs"):
(host, port,
self.db) = ConfigReader.read_section_keylist(section_name,
["host", "port", "db"])
self.__pool__ = ConnectionPool(host=host, port=port, db=self.db)
self._redis_conn = Redis(connection_pool=self.__pool__)
base_info("Redis连接%s创建成功！[%s:%s db%s]" %
(section_name, host, port, self.db))
def __init__(self, num=1):
raw_url = ConfigReader.read_section_key("proxy", "raw_url")
self.vaild_urls = []
self.ip_infos = []
for i in range(0, num):
self.vaild_urls.append(raw_url)
ProxiesRequests.__init__(self, self.vaild_urls)
def __init__(self):
self.config = ConfigReader()
self.logger = logging.getLogger(__name__)
self.logger.debug("Loading config properties for AWS")
self.default_region = self.config.get_property(
self.config.SECTION_AWS_ACCOUNT,
self.config.OPTION_DEFAULT_REGION
)
def create(db):
'''新建空表操作'''
c1_sql = create_house_list_table.format(tablename=house_list_name)
db.execute(c1_sql)
ps_mday = ConfigReader.read_section_key("quota", "ps_mday")
if int(ps_mday) == int(time.localtime().tm_mday):
c2_sql = create_price_trend_table.format(tablename=price_trend_name)
db.execute(c2_sql)
def __init__(self):
# 获取配置文件
(self.orderno, self.secret,
self.ip_port) = ConfigReader.read_section_keylist(
"proxy", ["orderno", "secret", "ip_port"])
self.__timestamp = str(int(time.time()))
self.__string = "orderno={orderno},secret={secret},timestamp={timestamp}".format(
orderno=self.orderno,
secret=self.secret,
timestamp=self.__timestamp)
self.__string = self.__string.encode()
def refresh_count_table():
'''刷新所有表的数据量'''
bak_datetime = DateTime(
ConfigReader.read_section_key("backup", "start_date"))
now_datetime = DateTime(Time.ISO_date_str())
month_list = list()
for _ in range(0, now_datetime - bak_datetime + 1):
date_str = bak_datetime.date_str
month_str = date_str[:-2]
backup_count_table(bak_datetime.date_str, month_str in month_list)
bak_datetime += 1
month_list.append(month_str)
def __init__(self, section_name="database"):
import pymysql
from pymysql.err import IntegrityError
(host, port, user, passwd, db) = ConfigReader.read_section_keylist(
section_name, ["host", "port", "user", "passwd", "db"])
# 保护连接为私有成员
try:
self._conn = pymysql.connect(host=host,
port=int(port),
user=user,
passwd=passwd,
db=db,
charset='utf8')
except Exception:
db_fatal("数据库连接创建失败！[%s:%s@%s:%s?charset=%s/%s]" %
(user, passwd, host, port, "utf-8", db))
self.cur = self._conn.cursor()
self.IntegrityError = IntegrityError
db_info("数据库连接创建成功！[%s:%s@%s:%s?charset=%s/%s]" %
(user, passwd, host, port, "utf-8", db))
# -*- coding: utf-8 -*-
import os
from util.config import ConfigReader
# Lambda Functions about conf_path
# Return path lead to appointed config file.
conf_path_func = lambda config : '{}/config/{}.cfg'.format(os.getcwd(), config)
conf_kv_func = lambda load, *k, **kwargs: ConfigReader.read_section_key(conf_path_func(load.split('.')[0]), load.split('.')[1], *k, **kwargs)
# Basic Config Value.
REDIS_CFG = conf_kv_func("sys.redis", all=True)
MYSQL_CFG = conf_kv_func("sys.database", all=True)
PROXY_CFG = conf_kv_func("sys.proxy", all=True)
REQUEST_CFG = conf_kv_func("sys.request", all=True)
BD_MAP_CFG = conf_kv_func("sys.bd_map", all=True)
EMAIL_CFG = conf_kv_func("sys.email", all=True)
LOGGER_CFG = conf_kv_func("sys.logger", all=True)