Example #1
0
def get_mysql_db(db_name, user=None, passwd=None, profile=None, host='localhost', port=3306):
    """
    建立MySQL连接
    :param db_name:
    :param user:
    :param passwd:
    :param profile:
    :param host:
    :param port:
    :return:
    """

    cached = getattr(get_mysql_db, 'cached', {})
    sig = '%s|%s|%s|%s|%s|%s' % (db_name, profile, host, port, user, passwd)
    if sig in cached:
        return cached[sig]

    cfg = load_yaml()
    if profile and profile in cfg:
        section = cfg[profile]
        host = section.get('host', 'localhost')
        port = int(section.get('port', '3306'))
        user = section.get('user', None)
        passwd = section.get('passwd', None)

    from MySQLdb.cursors import DictCursor
    import MySQLdb

    return MySQLdb.connect(host=host, port=port, user=user, passwd=passwd, db=db_name, cursorclass=DictCursor,
                           charset='utf8')
Example #2
0
def proc_crawler_settings(crawler):
    """
    读取配置文件,进行相应的设置
    """
    settings = crawler.settings

    config = conf.load_yaml()
    if 'scrapy' in config:
        for key, value in config['scrapy'].items():
            settings.set(key, value)
Example #3
0
def proc_crawler_settings(crawler):
    """
    读取配置文件,进行相应的设置
    """
    settings = crawler.settings

    config = conf.load_yaml()
    if 'scrapy' in config:
        for key, value in config['scrapy'].items():
            settings.set(key, value)
Example #4
0
    def generate():
        from conf import load_yaml

        def func(cfg_entry):
            return ProxyVerifier(url=cfg_entry['url'], status_codes=cfg_entry['status'],
                                 text_list=cfg_entry['contains'])

        config = load_yaml()
        try:
            return map(func, config['proxy-verifier'])
        except KeyError:
            return []
Example #5
0
    def generate():
        from conf import load_yaml

        def func(cfg_entry):
            return ProxyVerifier(url=cfg_entry['url'],
                                 status_codes=cfg_entry['status'],
                                 text_list=cfg_entry['contains'])

        config = load_yaml()
        try:
            return map(func, config['proxy-verifier'])
        except KeyError:
            return []
Example #6
0
def load_mongodb_conf():
    """
    Load MongoDB configurations
    :return:
    """
    cached_conf = load_mongodb_conf.conf

    if not cached_conf:
        conf_all = load_yaml()
        tmp = conf_all['mongodb'] if 'mongodb' in conf_all else []
        cached_conf = dict((item['profile'], item) for item in tmp)
        load_mongodb_conf.conf = cached_conf

    return cached_conf
Example #7
0
def load_mongodb_conf():
    """
    Load MongoDB configurations
    :return:
    """
    cached_conf = load_mongodb_conf.conf

    if not cached_conf:
        conf_all = load_yaml()
        tmp = conf_all['mongodb'] if 'mongodb' in conf_all else []
        cached_conf = dict((item['profile'], item) for item in tmp)
        load_mongodb_conf.conf = cached_conf

    return cached_conf
Example #8
0
def get_mysql_db(db_name,
                 user=None,
                 passwd=None,
                 profile=None,
                 host='localhost',
                 port=3306):
    """
    建立MySQL连接
    :param db_name:
    :param user:
    :param passwd:
    :param profile:
    :param host:
    :param port:
    :return:
    """

    cached = getattr(get_mysql_db, 'cached', {})
    sig = '%s|%s|%s|%s|%s|%s' % (db_name, profile, host, port, user, passwd)
    if sig in cached:
        return cached[sig]

    cfg = load_yaml()
    if profile and profile in cfg:
        section = cfg[profile]
        host = section.get('host', 'localhost')
        port = int(section.get('port', '3306'))
        user = section.get('user', None)
        passwd = section.get('passwd', None)

    from MySQLdb.cursors import DictCursor
    import MySQLdb

    return MySQLdb.connect(host=host,
                           port=port,
                           user=user,
                           passwd=passwd,
                           db=db_name,
                           cursorclass=DictCursor,
                           charset='utf8')