def rpter_config(rpter_name): rpter_name = "rpt_%s" % rpter_name # Get Xlsx File Config xlsx_conf = conf_kv_func("%s.xlsx" % rpter_name, all=True) # Get Recver Config recv_conf = conf_kv_func("%s.recv" % rpter_name, all=True) recv_conf['recv_mail'] = recv_conf['recv_mail'].split(',') return dict(xlsx=xlsx_conf, recv=recv_conf)
def __init__(self): # 获取配置文件 self._conf = conf_kv_func("sys.proxy", all=True) self.__timestamp = str(int(time.time())) self.__string = "orderno={},secret={},timestamp={}".format( self._conf["orderno"], self._conf["secret"], self.__timestamp) self.__string = self.__string.encode()
def etl_config(etl_name): etl_name = "etl_%s" % etl_name # Get system config. sys_conf = conf_kv_func("%s.sys_config" % etl_name, all=True) return dict(sys_conf=sys_conf)
def ziroom_extra(project_name, rid, rtn_data): '''ziroom_extra Ziroom Extra func. ''' logger = LogBase(project_name, "ziroom_extra") logger.debug("Before Extra =>", data=rtn_data) # Extra func for house code. try: end = int(rtn_data['house_code'].split('_')[1]) room_num = int(findall(r"([0-9])室[0-9]厅", rtn_data['house_type'])[0]) except Exception: pass else: if room_num > 1: rds = RedisController( int(conf_kv_func("ziroom.sys_config", all=True)['redis_db']), project_name) for idx in range(1, room_num + 1): rds.__update_dict_to_redis__( rid - end + idx, {"house_id": str(rid - end + idx)}) # Extra func for price. try: price_dict = dict() price, price_dict = get_price_from_png(rtn_data["price"], price_dict, project_name) rtn_data["price"] = price except Exception: pass # Extra func for payment. try: payment_rtn_list = list() for payment in rtn_data["paymentlist"]: payment_rtn = dict() for k, v in zip(payment.keys(), payment.values()): if k == "period": payment_rtn["period"] = v else: payment_rtn[k], price_dict = get_price_from_png( v, price_dict, project_name) payment_rtn_list.append(payment_rtn) rtn_data["paymentlist"] = payment_rtn_list except Exception: pass logger.debug("After Extra =>", data=rtn_data) return rtn_data
def crawler_config(crawler_name): # Get system config. sys_conf = conf_kv_func("%s.sys_config" % crawler_name, all=True) # Get list crawler config. list_crawler = conf_kv_func("%s.list_crawler" % crawler_name, all=True) list_parser = conf_kv_func("%s.list_parser" % crawler_name, all=True) # Get compile config for `re` module. compiles = conf_kv_func("%s.compiles" % crawler_name, all=True) # Get detail crawler config list. You may have many different configure to get detail infos. idx = 0 detail_crawlers = list() detail_parsers = list() while True: try: idx += 1 detail_crawler = conf_kv_func("%s.detail_crawler%d" % (crawler_name, idx), all=True) detail_parser = conf_kv_func("%s.detail_parser%d" % (crawler_name, idx), all=True) except KeyError: print("Load all detail crawler config succeed.") break else: detail_crawlers.append(detail_crawler) detail_parsers.append(detail_parser) return dict(sys_conf=sys_conf, list_crawler=list_crawler, list_parser=list_parser, detail_crawlers=detail_crawlers, detail_parsers=detail_parsers, compiles=compiles)
def __init__(self, num=1, project_name="test"): raw_url = conf_kv_func("sys.proxy", "raw_url") self.vaild_urls = [raw_url] * num self.ip_infos = [] ProxiesRequests.__init__(self, self.vaild_urls, project_name) self.info("Here is debug urls =>", urls=self.vaild_urls)