def rpter_config(rpter_name):
rpter_name = "rpt_%s" % rpter_name
# Get Xlsx File Config
xlsx_conf = conf_kv_func("%s.xlsx" % rpter_name, all=True)
# Get Recver Config
recv_conf = conf_kv_func("%s.recv" % rpter_name, all=True)
recv_conf['recv_mail'] = recv_conf['recv_mail'].split(',')
return dict(xlsx=xlsx_conf, recv=recv_conf)
def __init__(self):
# 获取配置文件
self._conf = conf_kv_func("sys.proxy", all=True)
self.__timestamp = str(int(time.time()))
self.__string = "orderno={},secret={},timestamp={}".format(
self._conf["orderno"], self._conf["secret"], self.__timestamp)
self.__string = self.__string.encode()
def etl_config(etl_name):
etl_name = "etl_%s" % etl_name
# Get system config.
sys_conf = conf_kv_func("%s.sys_config" % etl_name, all=True)
return dict(sys_conf=sys_conf)
def ziroom_extra(project_name, rid, rtn_data):
'''ziroom_extra
Ziroom Extra func.
'''
logger = LogBase(project_name, "ziroom_extra")
logger.debug("Before Extra =>", data=rtn_data)
# Extra func for house code.
try:
end = int(rtn_data['house_code'].split('_')[1])
room_num = int(findall(r"([0-9])室[0-9]厅", rtn_data['house_type'])[0])
except Exception:
pass
else:
if room_num > 1:
rds = RedisController(
int(conf_kv_func("ziroom.sys_config", all=True)['redis_db']),
project_name)
for idx in range(1, room_num + 1):
rds.__update_dict_to_redis__(
rid - end + idx, {"house_id": str(rid - end + idx)})
# Extra func for price.
try:
price_dict = dict()
price, price_dict = get_price_from_png(rtn_data["price"], price_dict,
project_name)
rtn_data["price"] = price
except Exception:
pass
# Extra func for payment.
try:
payment_rtn_list = list()
for payment in rtn_data["paymentlist"]:
payment_rtn = dict()
for k, v in zip(payment.keys(), payment.values()):
if k == "period":
payment_rtn["period"] = v
else:
payment_rtn[k], price_dict = get_price_from_png(
v, price_dict, project_name)
payment_rtn_list.append(payment_rtn)
rtn_data["paymentlist"] = payment_rtn_list
except Exception:
pass
logger.debug("After Extra =>", data=rtn_data)
return rtn_data
def crawler_config(crawler_name):
# Get system config.
sys_conf = conf_kv_func("%s.sys_config" % crawler_name, all=True)
# Get list crawler config.
list_crawler = conf_kv_func("%s.list_crawler" % crawler_name, all=True)
list_parser = conf_kv_func("%s.list_parser" % crawler_name, all=True)
# Get compile config for `re` module.
compiles = conf_kv_func("%s.compiles" % crawler_name, all=True)
# Get detail crawler config list. You may have many different configure to get detail infos.
idx = 0
detail_crawlers = list()
detail_parsers = list()
while True:
try:
idx += 1
detail_crawler = conf_kv_func("%s.detail_crawler%d" %
(crawler_name, idx),
all=True)
detail_parser = conf_kv_func("%s.detail_parser%d" %
(crawler_name, idx),
all=True)
except KeyError:
print("Load all detail crawler config succeed.")
break
else:
detail_crawlers.append(detail_crawler)
detail_parsers.append(detail_parser)
return dict(sys_conf=sys_conf,
list_crawler=list_crawler,
list_parser=list_parser,
detail_crawlers=detail_crawlers,
detail_parsers=detail_parsers,
compiles=compiles)
def __init__(self, num=1, project_name="test"):
raw_url = conf_kv_func("sys.proxy", "raw_url")
self.vaild_urls = [raw_url] * num
self.ip_infos = []
ProxiesRequests.__init__(self, self.vaild_urls, project_name)
self.info("Here is debug urls =>", urls=self.vaild_urls)