def initialize_info(self, user_config):
"""初始化爬虫信息"""
self.weibo = []
self.user = UserInfo()
self.user_config = user_config
self.got_count = 0
self.weibo_id_list = []
def bassCreateUser(self, username, userId):
user = UserInfo.UserInfo()
user.name = username
user.id = userId
user = user.__dict__
# 打印字典
print(user)
# 字典转化为json
userJson = json.dumps(user)
userDao = UserDao.UserDao()
userDict = userDao.baasCreateUser(userJson)
userInfo = UserInfo.UserInfo()
userInfo.userId, userInfo.publicKey, userInfo.privateKey, userInfo.address = userDict[
"id"], userDict['basePublicKey'], userDict[
'basePrivateKey'], userDict['baseAccountAddress']
return userInfo
def get_user_info(self):
"""获取用户信息"""
params = {'containerid': '100505' + str(self.user_config['user_id'])}
js = self.get_json(params)
if js['ok']:
info = js['data']['userInfo']
user_info = OrderedDict()
user_info['user_id'] = self.user_config['user_id']
user_info['screen_name'] = info.get('screen_name', '')
user_info['gender'] = "女" if js['data']['userInfo']['gender'] == "f" else "男"
params = {
'containerid':
'230283' + str(self.user_config['user_id']) + '_-_INFO'
}
zh_list = [
u'生日', u'所在地', u'小学', u'初中', u'高中', u'大学', u'公司', u'注册时间',
u'阳光信用'
]
en_list = [
'birthday', 'location', 'education', 'education', 'education',
'education', 'company', 'registration_time', 'sunshine'
]
for i in en_list:
user_info[i] = ''
js = self.get_json(params)
if js['ok']:
cards = js['data']['cards']
if isinstance(cards, list) and len(cards) > 1:
card_list = cards[0]['card_group'] + cards[1]['card_group']
for card in card_list:
if card.get('item_name') in zh_list:
user_info[en_list[zh_list.index(
card.get('item_name'))]] = card.get(
'item_content', '')
user_info['statuses_count'] = info.get('statuses_count', 0)
user_info['followers_count'] = info.get('followers_count', 0)
user_info['follow_count'] = info.get('follow_count', 0)
user_info['description'] = info.get('description', '')
user_info['profile_url'] = info.get('profile_url', '')
user_info['profile_image_url'] = info.get('profile_image_url', '')
user_info['avatar_hd'] = info.get('avatar_hd', '')
user_info['urank'] = info.get('urank', 0)
user_info['mbrank'] = info.get('mbrank', 0)
user_info['verified'] = info.get('verified', False)
user_info['verified_type'] = info.get('verified_type', -1)
user_info['verified_reason'] = info.get('verified_reason', '')
user = self.standardize_info(user_info)
self.user = UserInfo(**dict(user))
self.user_to_database()
return user
async def join(users: UserI):
userlist=list(users)
user_id = userlist[1][1]
user_pw = userlist[2][1]
user_name = userlist[3][1]
user_age = userlist[4][1]
user = UserInfo()
user.user_id = user_id
user.user_pw = user_pw
user.user_name = user_name
user.user_age = user_age
session.add(user)
session.commit()
return "회원가입완료!"
def get(self):
self.response.headers['Content-Type'] = 'text/html'
user = users.get_current_user()
current_folder = self.request.get('current_folder')
folder = None
if user:
user_key = ndb.Key('UserInfo', user.email())
user_info = user_key.get()
if user_info is None:
user_info = UserInfo(id=user.email())
user_info.email = user.email()
folder_info = FolderInfo(id=user.email() + "/")
folder_info.name = "/"
user_info.folder = folder_info.key
folder_info.put()
user_info.put()
time.sleep(1)
if len(current_folder) == 0:
folder = user_info.folder.get()
else:
folder_key = ndb.Key('FolderInfo',
user.email() + current_folder)
folder = folder_key.get()
url = users.create_logout_url(self.request.uri)
url_string = 'logout'
else:
url = users.create_login_url(self.request.uri)
url_string = 'login'
template_values = {
'url': url,
'url_string': url_string,
'user': user,
'upload_url': blobstore.create_upload_url('/upload'),
'message': self.request.get('message'),
'current_folder': folder
}
template = JINJA_ENVIRONMENT.get_template('main.html')
self.response.write(template.render(template_values))
MainPage.obj = self
def findUserInfoByAddress(self, address):
sql = """
select id,name,address,publicKey,privateKey from user a left join user_info b on a.id = b.userId where address = %s
"""
array = []
param = []
param.append(address)
try:
baseDao = BaseDao.BaseDao()
result = baseDao.execteGetOneSql(sql, param)
user = UserInfo.UserInfo()
user.id = result[0]
user.name = result[1]
user.address = result[2]
user.privateKey = result[4]
user.publicKey = result[3]
return user
except Exception as e:
print(e)
def user_info():
"""
Grab user's profile information into the database
user sign up page, user must log in successfully before getting to this page
"""
if not is_loggedin():
return redirect(url_for('login'))
form=UserInfoForm()
if request.method=='POST':
userInfo=UserInfo(form.nickName.data, form.email.data, form.phone.data, form.city.data,
form.state.data, form.zipcode.data, form.education.data, form.sports.data,form.arts.data,
form.travel.data,form.music.data,form.reading.data,form.gardening.data, form.nature.data,
form.snowboard.data,form.food.data)
db.session.add(userInfo) #add returned data to user table in the database
db.session.commit()
return redirect(url_for('home'))
elif request.method=='GET':
return render_template('user_info.html',form=form)
def getBossAccount(self, customerName):
sql = """
select id,name,address,publicKey,privateKey from user a left join user_info b on a.id = b.userId where customerName = %s
and b.isBoss=0
"""
array = []
param = []
param.append(customerName)
try:
baseDao = BaseDao.BaseDao()
result = baseDao.execteGetOneSql(sql, param)
user = UserInfo.UserInfo()
user.id = result[0]
user.name = result[1]
user.address = result[2]
user.privateKey = result[4]
user.publicKey = result[3]
return user
except Exception as e:
print(e)
def findUserInfo(self, customerName):
sql = """
select id,name,address,publicKey,privateKey from user a left join user_info b on a.id = b.userId where customerName = %s
and b.isBoss=1
"""
array = []
param = []
param.append(customerName)
try:
result = baseDao.executeGetAllSql(sql, param)
for row in result:
user = UserInfo.UserInfo()
user.id = row[0]
user.name = row[1]
user.address = row[2]
user.privateKey = row[4]
user.publicKey = row[3]
array.append(user)
return array
except Exception as e:
print(e)
def settleTest(address,amount):
userController = NewUserController.NewUserController()
userInfo = UserInfo.UserInfo()
list=[]
settleObject =NewSettleObject.NewSettleObject()
settleObject.amount = amount
settleObject.ownerAccount = address
userInfo = userController.findUserInfoByAddress(settleObject.ownerAccount)
list.append(userInfo)
settleObject.userPrivateKey = userInfo.privateKey
userAssetArray = userController.findAssetId(userInfo)
userAssetStr = ""
first= True
for each in userAssetArray:
if first:
userAssetStr = userAssetStr+each.assetAddress
first=False
else:
userAssetStr = userAssetStr+"," + each.assetAddress
settleObject.srcAsset = userAssetStr
userController.settle(settleObject)
def __init__(self, user_id_list, config=None):
"""Weibo类初始化"""
if not config:
config = weibo_config
config['user_id_list'] = user_id_list
self.validate_config(config)
self.filter = config[
'filter'] # 取值范围为0、1,程序默认值为0,代表要爬取用户的全部微博,1代表只爬取用户的原创微博
since_date = config['since_date']
if isinstance(since_date, int):
since_date = date.today() - timedelta(since_date)
since_date = str(since_date)
self.since_date = since_date # 起始时间，即爬取发布日期从该值到现在的微博，形式为yyyy-mm-dd
self.start_page = config.get('start_page',
1) # 开始爬的页，如果中途被限制而结束可以用此定义开始页码
self.write_mode = config[
'write_mode'] # 结果信息保存类型，为list形式，可包含csv、mongo和mysql三种类型
self.original_pic_download = config[
'original_pic_download'] # 取值范围为0、1, 0代表不下载原创微博图片,1代表下载
self.retweet_pic_download = config[
'retweet_pic_download'] # 取值范围为0、1, 0代表不下载转发微博图片,1代表下载
self.original_video_download = config[
'original_video_download'] # 取值范围为0、1, 0代表不下载原创微博视频,1代表下载
self.retweet_video_download = config[
'retweet_video_download'] # 取值范围为0、1, 0代表不下载转发微博视频,1代表下载
self.result_dir_name = config.get(
'result_dir_name', 0) # 结果目录名，取值为0或1，决定结果文件存储在用户昵称文件夹里还是用户id文件夹里
cookie = config.get('cookie') # 微博cookie，可填可不填
user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36'
self.headers = {'User_Agent': user_agent, 'Cookie': cookie}
self.mysql_config = config.get('mysql_config') # MySQL数据库连接配置，可以不填
query_list = config.get('query_list') or []
if isinstance(query_list, str):
query_list = query_list.split(',')
self.query_list = query_list
if not isinstance(user_id_list, list):
if not os.path.isabs(user_id_list):
user_id_list = os.path.split(
os.path.realpath(__file__))[0] + os.sep + user_id_list
self.user_config_file_path = user_id_list # 用户配置文件路径
user_config_list = self.get_user_config_list(user_id_list)
else:
self.user_config_file_path = ''
user_config_list = [{
'user_id': user_id,
'since_date': self.since_date,
'query_list': query_list
} for user_id in user_id_list]
self.user_config_list = user_config_list # 要爬取的微博用户的user_config列表
self.user_config = {} # 用户配置,包含用户id和since_date
self.start_date = '' # 获取用户第一条微博时的日期
self.query = ''
self.user: UserInfo = UserInfo() # 存储目标微博用户信息
self.got_count = 0 # 存储爬取到的微博数
self.weibo = [] # 存储爬取到的所有微博信息
self.weibo_id_list = [] # 存储爬取到的所有微博id
self.proxies = requests.get(
"http://api.hailiangip.com:8422/api/getIp?type=1&num=1&pid=&unbindTime=600&cid=&orderId=O21042810412537647150&time=1619577728&sign=d79086f5b8ba9dbe1a17e5b710b77032&noDuplicate=1&dataType=0&lineSeparator=0&singleIp="
).json()['data'][0]
logger.info({
"http": f"http://{self.proxies['ip']}:{self.proxies['port']}",
"https": f"https://{self.proxies['ip']}:{self.proxies['port']}",
})
