Ejemplo n.º 1
0
def get_seed_ids():
    """
    操作weibo_search_data表,获取待爬取用户id队列
    :return:
    """
    truncate_sql = 'truncate table weibo_sinausers_cache'
    insert_sql = 'insert into weibo_sinausers_cache (select se_userid from weibo_search_data where is_new = 1 ' \
                 'and se_sourcetype=\'新浪微博\' group by se_userid)'
    delelte_sql = 'delete from weibo_sinausers_cache where dsu_id in (select su_id from weibo_sina_users)'
    update_sql = 'update weibo_search_data set is_new = 0 where is_new = 1 and se_sourcetype = \'新浪微博\''
    select_sql = 'select dsu_id from weibo_sinausers_cache'
    con = db_connect.get_con()
    db_connect.db_dml(con, truncate_sql)
    print('-----------临时表已清空--------------')
    db_connect.db_dml(con, insert_sql)
    print('-----------临时表数据插入完成--------------')
    db_connect.db_dml(con, delelte_sql)
    print('-----------临时表已去重--------------')
    db_connect.db_dml(con, update_sql)
    print('-----------search表已更新--------------')
    rs = db_connect.db_queryall(con, select_sql)
    print('获取到{num}条需要爬取的id'.format(num=len(rs)))
    db_connect.db_close(con)
    ids = []
    for r in rs:
        ids.append(r[0])
    return ids
Ejemplo n.º 2
0
def add_search_cont(search_list):
    save_sql = 'insert into weibo_search (mk_primary,mid,murl,create_time,praise_count,repost_count,comment_count,' \
               'content,device,user_id,username,uheadimage,user_home,keyword) values(:mk_primary, :mid, ' \
               ':murl, :create_time, :praise_count,:repost_count, :comment_count, :content, :device, ' \
               ':user_id, :username,:uheadimage, :user_home, :keyword)'

    con = db_connect.get_con()

    for search_cont in search_list:
        search_info = {
            'mk_primary': search_cont.mk_primary,
            'mid': search_cont.mid,
            'murl': search_cont.murl,
            'create_time': search_cont.create_time,
            'praise_count': search_cont.praise_count,
            'repost_count': search_cont.repost_count,
            'comment_count': search_cont.comment_count,
            'content': search_cont.content,
            'device': search_cont.device,
            'user_id': search_cont.user_id,
            'username': search_cont.username,
            'uheadimage': search_cont.uheadimage,
            'user_home': search_cont.user_home,
            'keyword': search_cont.keyword
        }
        try:
            db_connect.db_dml_parms(con, save_sql, search_info)
        except Exception as why:
            print('插入出错,具体原因为:{why}'.format(why=why))
            print(search_info.__dict__)
    db_connect.db_close(con)
Ejemplo n.º 3
0
def add_search_cont(search_list):
    save_sql = 'insert into weibo_search (mk_primary,mid,murl,create_time,praise_count,repost_count,comment_count,' \
               'content,device,user_id,username,uheadimage,user_home,keyword) values(:mk_primary, :mid, ' \
               ':murl, :create_time, :praise_count,:repost_count, :comment_count, :content, :device, ' \
               ':user_id, :username,:uheadimage, :user_home, :keyword)'

    con = db_connect.get_con()

    for search_cont in search_list:
        search_info = {
            'mk_primary': search_cont.mk_primary,
            'mid': search_cont.mid,
            'murl': search_cont.murl,
            'create_time': search_cont.create_time,
            'praise_count': search_cont.praise_count,
            'repost_count': search_cont.repost_count,
            'comment_count': search_cont.comment_count,
            'content': search_cont.content,
            'device': search_cont.device,
            'user_id': search_cont.user_id,
            'username': search_cont.username,
            'uheadimage': search_cont.uheadimage,
            'user_home': search_cont.user_home,
            'keyword': search_cont.keyword
        }
        try:
            db_connect.db_dml_parms(con, save_sql, search_info)
        except Exception as why:
            print('插入出错,具体原因为:{why}'.format(why=why))
            print(search_info.__dict__)
    db_connect.db_close(con)
Ejemplo n.º 4
0
def get_login_info():
    con = db_connect.get_con()
    sql = 'SELECT * FROM (SELECT * FROM sina_login_infodangbantest order by dbms_random.value) WHERE rownum =1'
    r = db_connect.db_queryone(con, sql)
    db_connect.db_close(con)
    login_name = r[2]
    pass_word = r[3]
    return login_name, pass_word
Ejemplo n.º 5
0
def save(user, mid, post_time, source, reposts_count, comments_count, root_url):
    """
    :param user: 用户对象
    :param mid: 微博id
    :param post_time: 发表时间
    :param source: 网页源码
    :param reposts_count: 转发数
    :param comments_count: 评论数
    :param root_url: 源微博URL
    :return: 返回的结果用于判断是否需要进行微博扩散的抓取
    """
    conn = db_connect.get_con()
    select_sql = "select * from weibo_spread_original where status_mid = '" + str(mid) + "'"
    child_sql = "select count(*) from weibo_spread_other where original_status_id = '" + str(mid) + "'"
    r = db_connect.db_queryall(conn, select_sql)
    rc = db_connect.db_queryall(conn, child_sql)

    # 如果数据库存在源微博和它的一些转发信息,我们就认为它不必抓取了
    if len(r) > 0 and rc[0][0] > 0:
        print('关于此条微博的扩散信息已经存于数据库中')
        db_connect.db_close(conn)
        return False

    insert_sql = 'insert into weibo_spread_original (user_id,user_screenname,user_province,user_city,user_location,' \
                 'user_description,user_url,user_profileimageurl,user_gender,user_followerscount,user_friendscount,' \
                 'user_statusescount,user_createdat,user_verifiedtype,user_verifiedreason,status_createdat,' \
                 'status_mid,status_source,status_repostscount,status_commentscount,status_url) ' + " values (" \
                 ":user_id,:user_screenname,:user_province,:user_city,:user_location,:user_description,:user_url," \
                 ":user_profileimageurl,:user_gender,:user_followerscount,:user_friendscount,:user_statusescount," \
                 ":user_createdat,:user_verifiedtype,:user_verifiedreason,:status_createdat,:status_mid," \
                 ":status_source,:status_repostscount,:status_commentscount,:status_url)"
    args = {
        'user_id': user.id,
        'user_screenname': user.screen_name,
        'user_province': user.province,
        'user_city': user.city,
        'user_location': user.location,
        'user_description': user.description.encode('gbk', 'ignore').decode('gbk'),
        'user_url': user.blog_url,
        'user_profileimageurl': user.headimg_url,
        'user_followerscount': user.followers_count,
        'user_friendscount': user.friends_count,
        'user_statusescount': user.status_count,
        'user_createdat': user.register_time,
        'user_verifiedtype': user.verify_type,
        'user_verifiedreason': user.verify_info.encode('gbk', 'ignore').decode('gbk'),
        'user_gender': user.gender,
        'status_createdat': post_time,
        'status_mid': mid,
        'status_source': source,
        'status_repostscount': reposts_count,
        'status_commentscount': comments_count,
        'status_url': root_url,
    }
    db_connect.db_dml_parms(conn, insert_sql, args)
    db_connect.db_close(conn)
    return True
Ejemplo n.º 6
0
def update_weibo_repost(mid, reposts_count):
    sql = 'select se_repost_count from weibo_search_data where se_mid = :mid'
    args = {'mid': str(mid)}
    con = db_connect.get_con()
    rs = db_connect.db_queryone_params(con, sql, args)
    if reposts_count != rs[0]:
        update_sql = 'update weibo_search_data set se_repost_count = :reposts_count where se_mid = :mid'
        update_args = {'mid': mid, 'reposts_count': reposts_count}
        db_connect.db_dml_parms(con, update_sql, update_args)
    db_connect.db_close(con)
Ejemplo n.º 7
0
def update_weibo_repost(mid, reposts_count):
    sql = 'select se_repost_count from weibo_search_data where se_mid = :mid'
    args = {'mid': str(mid)}
    con = db_connect.get_con()
    rs = db_connect.db_queryone_params(con, sql, args)
    if reposts_count != rs[0]:
        update_sql = 'update weibo_search_data set se_repost_count = :reposts_count where se_mid = :mid'
        update_args = {'mid': mid, 'reposts_count': reposts_count}
        db_connect.db_dml_parms(con, update_sql, update_args)
    db_connect.db_close(con)
Ejemplo n.º 8
0
def get_crawl_urls():
    """
    :return: is_crawled = 0的字段,即需要进行扩散分析的字段
    """
    sql = 'select se_userid,se_sid, se_mid from weibo_search_data where is_crawled = 0 and ' \
          'se_sourcetype = \'新浪微博\''
    con = db_connect.get_con()
    rs = db_connect.db_queryall(con, sql)
    db_connect.db_close(con)
    datas = []
    for r in rs:
        data = {'url': 'http://weibo.com/' + r[0] + '/' + r[1], 'mid': r[2]}
        datas.append(data)
    return datas
Ejemplo n.º 9
0
def save(sos):
    ins_count = 0
    conn = db_connect.get_con()
    insert_sql = 'insert into weibo_spread_other (user_id,user_screenname,user_province,user_city,user_location,' \
                 'user_description,user_url,user_profileimageurl,user_gender,user_followerscount,user_friendscount,' \
                 'user_statusescount,user_createdat,user_verifiedtype,user_verifiedreason,status_createdat,' \
                 'status_mid,status_source,status_repostscount,status_commentscount,upper_user_id,original_status_id,' \
                 'status_url) ' + " values (:user_id,:user_screenname,:user_province,:user_city,:user_location," \
                 ":user_description,:user_url,:user_profileimageurl,:user_gender,:user_followerscount," \
                 ":user_friendscount,:user_statusescount,:user_createdat,:user_verifiedtype,:user_verifiedreason," \
                 ":status_createdat,:status_mid,:status_source,:status_repostscount,:status_commentscount," \
                 ":upper_user_id,:original_status_id,:status_url)"
    for item in sos:
        if item.verify_type == '':
            item.verify_type = 0
        try:
            args = {
                'user_id': item.id,
                'user_url': item.blog_url,
                'user_profileimageurl': item.headimg_url,
                'user_screenname': item.screen_name.encode('gbk', 'ignore').decode('gbk'),
                'user_province': item.province.encode('gbk', 'ignore').decode('gbk'),
                'user_city': item.city.encode('gbk', 'ignore').decode('gbk'),
                'user_location': item.location.encode('gbk', 'ignore').decode('gbk'),
                'user_description': item.description.encode('gbk', 'ignore').decode('gbk'),
                'user_gender': item.gender.encode('gbk', 'ignore').decode('gbk'),
                'user_verifiedreason': item.verify_info.encode('gbk', 'ignore').decode('gbk'),
                'status_source': item.device.encode('gbk', 'ignore').decode('gbk'),
                'user_followerscount': int(item.followers_count),
                'user_friendscount': int(item.friends_count),
                'user_statusescount': int(item.status_count),
                'status_repostscount': int(item.reposts_count),
                'status_commentscount': int(item.comments_count),
                'user_verifiedtype': item.verify_type,
                'user_createdat': item.register_time,
                'status_createdat': item.status_post_time,
                'status_mid': item.mid,
                'upper_user_id': item.upper_user_id,
                'original_status_id': item.original_status_id,
                'status_url': item.status_url,
            }
            db_connect.db_dml_parms(conn, insert_sql, args)
        except Exception as why:
            print(item.__dict__)
            print(why)
        else:
            ins_count += 1
    print('一共插入了{ins}条数据'.format(ins=ins_count))
    db_connect.db_close(conn)
Ejemplo n.º 10
0
def get_crawl_urls():
    """
    :return: is_crawled = 0的字段,即需要进行扩散分析的字段
    """
    # 以下代码是为了测试反爬虫机制注释掉的
    sql = 'select se_userid,se_sid, se_mid from weibo_search_data where is_crawled = 0 and ' \
           'se_sourcetype = \'新浪微博\' order by se_createtime desc'

    # sql = 'select se_userid,se_sid, se_mid, se_content from weibo_search_data where is_new = 1 and ' \
    #       'se_sourcetype = \'新浪微博\' order by se_createtime desc'
    con = db_connect.get_con()
    rs = db_connect.db_queryall(con, sql)
    db_connect.db_close(con)
    datas = []
    for r in rs:
        data = {'url': 'http://weibo.com/' + r[0] + '/' + r[1], 'mid': r[2]}
        datas.append(data)
    return datas
Ejemplo n.º 11
0
def get_crawl_urls():
    """
    :return: is_crawled = 0的字段,即需要进行扩散分析的字段
    """
    # 以下代码是为了测试反爬虫机制注释掉的
    sql = 'select se_userid,se_sid, se_mid from weibo_search_data where is_crawled = 0 and ' \
           'se_sourcetype = \'新浪微博\' order by se_createtime desc'

    # sql = 'select se_userid,se_sid, se_mid, se_content from weibo_search_data where is_new = 1 and ' \
    #       'se_sourcetype = \'新浪微博\' order by se_createtime desc'
    con = db_connect.get_con()
    rs = db_connect.db_queryall(con, sql)
    db_connect.db_close(con)
    datas = []
    for r in rs:
        data = {'url': 'http://weibo.com/' + r[0] + '/' + r[1], 'mid': r[2]}
        datas.append(data)
    return datas
Ejemplo n.º 12
0
def save_users(users):
    save_sql = 'insert into weibo_sina_users (su_id,su_screen_name,su_province,su_city,su_description,su_headimg_url,' \
               'su_blog_url,su_domain_name,su_gender,su_friends_count,su_followers_count,su_statuses_count,' \
               'su_gender_prefer,su_birthday,su_blood_type,su_contact_info,su_work_info,su_educate_info,' \
               'su_owntag_info,su_register_time,su_verifytype,su_verifyinfo) values(:su_id, :su_screen_name, ' \
               ':su_province, :su_city, :su_description,:su_headimg_url, :su_blog_url, :su_domain_name, :su_gender, ' \
               ':su_friends_count, :su_followers_count,:su_status_count, :su_gender_prefer, :su_birthday, ' \
               ':su_blood_type, :su_contact_info, :su_work_info,:su_educate_info, :su_owntag_info, :su_register_time,' \
               ':su_verifytype, :su_verifyinfo)'

    con = db_connect.get_con()

    datas = []
    for user in users:
        if user.id == '':
            continue
        user_info = {
            'su_id': user.id,
            'su_screen_name': user.screen_name,
            'su_province': user.province,
            'su_city': user.city,
            'su_description': user.description,
            'su_headimg_url': user.headimg_url,
            'su_blog_url': user.blog_url,
            'su_domain_name': user.domain_name,
            'su_gender': user.gender,
            'su_gender_prefer': user.gender_prefer,
            'su_friends_count': int(user.friends_count),
            'su_followers_count': int(user.followers_count),
            'su_status_count': int(user.status_count),
            'su_birthday': user.birthday,
            'su_blood_type': user.blood_type,
            'su_contact_info': user.contact_info,
            'su_work_info': user.work_info,
            'su_educate_info': user.educate_info,
            'su_owntag_info': user.owntag_info,
            'su_register_time': user.register_time,
            'su_verifytype': int(user.verify_type),
            'su_verifyinfo': user.verify_info,
        }
        datas.append(user_info)
    db_connect.db_dml_many(con, save_sql, datas)
    db_connect.db_close(con)
Ejemplo n.º 13
0
def save(user, mid, post_time, source, reposts_count, comments_count, root_url):
    conn = db_connect.get_con()
    select_sql = "select * from weibo_spread_original where status_mid = '"+str(mid)+"'"
    r = db_connect.db_queryall(conn, select_sql)
    if len(r) > 0:
        print('已经存在了')
        db_connect.db_close(conn)
        return
    insert_sql = 'insert into weibo_spread_original (user_id,user_screenname,user_province,user_city,user_location,' \
                 'user_description,user_url,user_profileimageurl,user_gender,user_followerscount,user_friendscount,' \
                 'user_statusescount,user_createdat,user_verifiedtype,user_verifiedreason,status_createdat,' \
                 'status_mid,status_source,status_repostscount,status_commentscount,status_url) ' + " values (" \
                 ":user_id,:user_screenname,:user_province,:user_city,:user_location,:user_description,:user_url," \
                 ":user_profileimageurl,:user_gender,:user_followerscount,:user_friendscount,:user_statusescount," \
                 ":user_createdat,:user_verifiedtype,:user_verifiedreason,:status_createdat,:status_mid," \
                 ":status_source,:status_repostscount,:status_commentscount,:status_url)"
    args = {
        'user_id': user.id,
        'user_screenname': user.screen_name,
        'user_province': user.province,
        'user_city': user.city,
        'user_location': user.location,
        'user_description': user.description.encode('gbk', 'ignore').decode('gbk'),
        'user_url': user.blog_url,
        'user_profileimageurl': user.headimg_url,
        'user_followerscount': user.followers_count,
        'user_friendscount': user.friends_count,
        'user_statusescount': user.status_count,
        'user_createdat': user.register_time,
        'user_verifiedtype': user.verify_type,
        'user_verifiedreason': user.verify_info.encode('gbk', 'ignore').decode('gbk'),
        'user_gender': user.gender,
        'status_createdat': post_time,
        'status_mid': mid,
        'status_source': source,
        'status_repostscount': reposts_count,
        'status_commentscount': comments_count,
        'status_url': root_url,
    }
    db_connect.db_dml_parms(conn, insert_sql, args)
    db_connect.db_close(conn)
Ejemplo n.º 14
0
def update_weibo_url(mid):
    sql = "update weibo_search_data set is_crawled = 1 where se_mid = :mid"
    args = {'mid': str(mid)}
    con = db_connect.get_con()
    db_connect.db_dml_parms(con, sql, args)
    db_connect.db_close(con)
Ejemplo n.º 15
0
def update_weibo_url(mid):
    sql = "update weibo_search_data set is_crawled = 1 where se_mid = :mid"
    args = {'mid': str(mid)}
    con = db_connect.get_con()
    db_connect.db_dml_parms(con, sql, args)
    db_connect.db_close(con)