def backup_count_table(t, flag): '''备份统计表''' count_b1_sql = count_backup_sql.format(tablename=house_list_name, counttablename="%s_%s" % (house_list_name, t), columnname="run_date", runtime=t) count_b2_sql = count_backup_sql.format(tablename=price_trend_name, counttablename="%s_%s" % (price_trend_name, t[:-2]), columnname="run_month", runtime=t[:-2]) ps_mday = ConfigReader.read_section_key("quota", "ps_mday") try: db.execute(count_b1_sql) db_optor_info("【%s】房源日常备份成功!" % t) except Exception: db_optor_info("【%s】房源日常备份失败!" % (t)) try: if not flag: db.execute(count_b2_sql) db_optor_info("【-%s-】价格趋势备份成功!" % (t[:-2])) except Exception: db_optor_info("【-%s-】价格趋势备份失败!" % (t[:-2]))
def backup_table(db, t): '''备份缓存表操作''' b1_sql = rename_sql.format(fromname=house_list_name, toname="%s_%s" % (house_list_name, t)) count_b1_sql = count_backup_sql.format(tablename=house_list_name, counttablename=house_list_name, columnname="run_date", runtime=t) db.execute(count_b1_sql) db.execute(b1_sql) ps_mday = ConfigReader.read_section_key("quota", "ps_mday") if int(ps_mday) == int(time.localtime().tm_mday): b2_sql = rename_sql.format(fromname=price_trend_name, toname="%s_%s" % (price_trend_name, t[:-2])) count_b2_sql = count_backup_sql.format(tablename=price_trend_name, counttablename=price_trend_name, columnname="run_month", runtime=t[:-2]) db.execute(count_b2_sql) db.execute(b2_sql) create(db) truncate_redis()
def __init__(self, num=1): raw_url = ConfigReader.read_section_key("proxy", "raw_url") self.vaild_urls = [] self.ip_infos = [] for i in range(0, num): self.vaild_urls.append(raw_url) ProxiesRequests.__init__(self, self.vaild_urls)
def create(db): '''新建空表操作''' c1_sql = create_house_list_table.format(tablename=house_list_name) db.execute(c1_sql) ps_mday = ConfigReader.read_section_key("quota", "ps_mday") if int(ps_mday) == int(time.localtime().tm_mday): c2_sql = create_price_trend_table.format(tablename=price_trend_name) db.execute(c2_sql)
def refresh_count_table(): '''刷新所有表的数据量''' bak_datetime = DateTime( ConfigReader.read_section_key("backup", "start_date")) now_datetime = DateTime(Time.ISO_date_str()) month_list = list() for _ in range(0, now_datetime - bak_datetime + 1): date_str = bak_datetime.date_str month_str = date_str[:-2] backup_count_table(bak_datetime.date_str, month_str in month_list) bak_datetime += 1 month_list.append(month_str)
# -*- coding: utf-8 -*- import os from util.config import ConfigReader # Lambda Functions about conf_path # Return path lead to appointed config file. conf_path_func = lambda config : '{}/config/{}.cfg'.format(os.getcwd(), config) conf_kv_func = lambda load, *k, **kwargs: ConfigReader.read_section_key(conf_path_func(load.split('.')[0]), load.split('.')[1], *k, **kwargs) # Basic Config Value. REDIS_CFG = conf_kv_func("sys.redis", all=True) MYSQL_CFG = conf_kv_func("sys.database", all=True) PROXY_CFG = conf_kv_func("sys.proxy", all=True) REQUEST_CFG = conf_kv_func("sys.request", all=True) BD_MAP_CFG = conf_kv_func("sys.bd_map", all=True) EMAIL_CFG = conf_kv_func("sys.email", all=True) LOGGER_CFG = conf_kv_func("sys.logger", all=True)