def get_mysql_db(db_name, user=None, passwd=None, profile=None, host='localhost', port=3306): """ 建立MySQL连接 :param db_name: :param user: :param passwd: :param profile: :param host: :param port: :return: """ cached = getattr(get_mysql_db, 'cached', {}) sig = '%s|%s|%s|%s|%s|%s' % (db_name, profile, host, port, user, passwd) if sig in cached: return cached[sig] cfg = load_yaml() if profile and profile in cfg: section = cfg[profile] host = section.get('host', 'localhost') port = int(section.get('port', '3306')) user = section.get('user', None) passwd = section.get('passwd', None) from MySQLdb.cursors import DictCursor import MySQLdb return MySQLdb.connect(host=host, port=port, user=user, passwd=passwd, db=db_name, cursorclass=DictCursor, charset='utf8')
def proc_crawler_settings(crawler): """ 读取配置文件,进行相应的设置 """ settings = crawler.settings config = conf.load_yaml() if 'scrapy' in config: for key, value in config['scrapy'].items(): settings.set(key, value)
def generate(): from conf import load_yaml def func(cfg_entry): return ProxyVerifier(url=cfg_entry['url'], status_codes=cfg_entry['status'], text_list=cfg_entry['contains']) config = load_yaml() try: return map(func, config['proxy-verifier']) except KeyError: return []
def load_mongodb_conf(): """ Load MongoDB configurations :return: """ cached_conf = load_mongodb_conf.conf if not cached_conf: conf_all = load_yaml() tmp = conf_all['mongodb'] if 'mongodb' in conf_all else [] cached_conf = dict((item['profile'], item) for item in tmp) load_mongodb_conf.conf = cached_conf return cached_conf