def __init__(self):
self.__rootPath = Helper().getRootPath()
self.cfp = configparser.ConfigParser()
self.cfp.read(self.__rootPath + '/conf/social.conf')
self.logger_access = Logger(self.__rootPath + '/conf/logging.conf',
'logger_access').createLogger()
self.logger_error = Logger(self.__rootPath + '/conf/logging.conf',
'logger_error').createLogger()
def __init__(self):
super(IdataAPI, self).__init__()
self.__apiToken = self.cfp.get('api', 'idata')
self.__rootPath = Helper().getRootPath()
self.__mongo_user = urllib.parse.quote_plus(self.cfp.get('mongodb_idata','user'))
self.__mongo_pwd = urllib.parse.quote_plus(self.cfp.get('mongodb_idata','pwd'))
self.__mongo_host = self.cfp.get('mongodb','host')
self.__mongo_port = self.cfp.get('mongodb','port')
self.__idata_url = self.cfp.get('idata_platform','url_base')
self.__mongo_uri = 'mongodb://' + self.__mongo_user + ':' + self.__mongo_pwd + '@' + self.__mongo_host + ':' + self.__mongo_port + '/' + 'idata'
self.client = MongoClient(self.__mongo_uri)
def __init__(self):
super(SocialWeixinAPI, self).__init__()
self.__apiToken = self.cfp.get('api', 'weibo')
self.__rootPath = Helper().getRootPath()
self.__mongo_user = urllib.parse.quote_plus(self.cfp.get('mongodb_weixin','user'))
self.__mongo_pwd = urllib.parse.quote_plus(self.cfp.get('mongodb_weixin','pwd'))
self.__mongo_host = self.cfp.get('mongodb','host')
self.__mongo_port = self.cfp.get('mongodb','port')
self.__redis_host = self.cfp.get('redis','host')
self.__redis_port = self.cfp.get('redis','port')
self.__redis_db = self.cfp.get('redis','db')
self.__redisPool = redis.ConnectionPool(host=self.__redis_host, port=self.__redis_port, db=self.__redis_db)
self.r = redis.Redis(connection_pool=self.__redisPool)
self.__mongo_uri = 'mongodb://' + self.__mongo_user + ':' + self.__mongo_pwd + '@' + self.__mongo_host + ':' + self.__mongo_port + '/' + 'weixin'
self.client = MongoClient(self.__mongo_uri)
def __init__(self):
"""
constructor
"""
self.user_name = None
self.pass_word = None
self.user_uniqueid = None
self.user_nick = None
self.__rootPath = Helper().getRootPath()
#self.logger = Logger(self.__rootPath + '/conf/logging.conf', 'simpleExample').createLogger()
self.logger_access = Logger(self.__rootPath + '/conf/logging.conf',
'logger_access').createLogger()
self.logger_error = Logger(self.__rootPath + '/conf/logging.conf',
'logger_error').createLogger()
self.session = requests.Session()
self.session.headers.update({
"User-Agent":
"Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0"
})
self.session.get("http://weibo.com/login.php")
return
from kafka import KafkaConsumer from subprocess import call,check_call,CalledProcessError import time import json from SocialAPI.Logger.BasicLogger import Logger from SocialAPI.Helper import Helper root_path = Helper().getRootPath() logger = Logger(root_path + '/conf/logging.conf','logger_kafka').createLogger() topics = ( 'idata_baidutieba_comment' ,'idata_baidutieba_post' ,'idata_baidutieba_reply' ,'idata_bilibili_comment' ,'idata_bilibili_video' ,'idata_dongqiudi_comment' ,'idata_dongqiudi_post' ,'idata_douyin_comment' ,'idata_douyin_video' ,'idata_hupu_comment' ,'idata_hupu_post' ,'idata_idataapi_article' ,'idata_iqiyi_comment' ,'idata_iqiyi_video' ,'idata_qqsport_comment' ,'idata_qqsport_post' ,'idata_tencent_comment' ,'idata_tencent_video' ,'idata_toutiao_comment' ,'idata_toutiao_news' ,'idata_toutiao_video'
def __init__(self):
super(SocialWeiboAPI, self).__init__()
self.__apiToken = self.cfp.get('api', 'weibo')
self.__rootPath = Helper().getRootPath()
import pandas as pd
from SocialAPI.SocialAPI.WeiboAPI import SocialWeiboAPI
from SocialAPI.Helper import Helper
from SocialAPI.Crawler import WeiBoCrawler
from SocialAPI.Model import Kol
import time
from datetime import datetime
from pymongo import MongoClient
if __name__ == '__main__':
myHelper = Helper()
weibo = SocialWeiboAPI()
session = weibo.createSession()
client = weibo.client
db = client.weibo
crawlTable = db.weibo_post_crawl
postTable = db.weibo_user_post
crawlDict = {}
startTime = weibo.getStrTime(-30)
startTimeStamp = weibo.getTimeStamp(startTime)
userDict = {}
userInfo = session.query(Kol.uid,Kol.username,Kol.pw).filter(Kol.status == 1, Kol.crawl_status==1).all()
#userInfo = session.query(Kol.uid,Kol.username,Kol.pw).filter(Kol.uid==2036201132).all()
for user in userInfo:
userDict[user[0]] = (user[1],user[2])
for uid in userDict.keys():