def robot_dynamic():
"""机器人发表动态
:return:
"""
scrapy = 'scrapy'
postgres = pgs.Pgs(host='localhost',
port=12432,
db_name=scrapy,
user=scrapy,
password=scrapy)
rows = postgres.fetch_all(planet_sql.find_random_music(), ())
row = rows[0]
music_name = row[1]
music_lyric = row[2]
comment = '{0}\n{1}'.format(music_name, music_lyric)
api = 'https://www.quanquanyuanyuan.cn/huodong/dog/api/tlmsg/post'
data = {
"comment": comment,
"photo_ids": [],
"sync_photo_to_album": False
}
resp = requests.post(api, json=data, headers=Planet.headers).json()
msg_id = resp.get('id')
logging.info('Robot dynamic msg_id %d', msg_id)
if msg_id:
table_id = row[0]
postgres.handler(planet_sql.update_music_count(), (table_id, ))
postgres.close()
def __init__(self):
# Postgres
host, port = 'localhost', 12432
db_name = 'scrapy'
self.postgres = pgs.Pgs(host=host, port=port, db_name=db_name, user=db_name, password=db_name)
# Redis
self.redis = rds.Rds(host=host, port=12379, db=2, password='******').redis_cli
def __init__(self,
_mps_id,
_biz,
_pass_ticket,
_app_msg_token,
_cookie,
_offset=0):
self.offset = _offset
self.mps_id = _mps_id
self.biz = _biz # 公众号标志
self.msg_token = _app_msg_token # 票据(非固定)
self.pass_ticket = _pass_ticket # 票据(非固定)
self.headers = {
'Cookie':
_cookie, # Cookie(非固定)
'User-Agent':
'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.132 '
}
wx_mps = 'wxmps' # 这里数据库、用户、密码一致(需替换成实际的)
self.postgres = pgs.Pgs(host='localhost',
port='12432',
db_name=wx_mps,
user=wx_mps,
password=wx_mps)
self.elastic = es.Es(host='localhost',
port=12900,
index='mp',
doc='article')
def __init__(self):
# PostgreSQL
host = 'localhost'
port = 12432
scrapy = 'scrapy'
self.postgres = pgs.Pgs(host=host,
port=port,
db_name=scrapy,
user=scrapy,
password=scrapy)
# Redis
self.redis = rds.Rds(host=host, port=12379, db=1,
password='******').redis_cli
class Planet(object):
log.Logger()
# Postgres配置
host = 'localhost'
pgs_port = 12432
pgs_db_name = 'planet'
pgs_user = pgs_db_name
pgs_pwd = pgs_db_name
postgres = pgs.Pgs(host=host,
port=pgs_port,
db_name=pgs_db_name,
user=pgs_user,
password=pgs_pwd,
singleton=False)
# Redis配置
rds_port = 12379
rds_pwd = 'redis6379'
redis = rds.Rds(host=host, port=rds_port, password=rds_pwd).redis_cli
my_hash = None
my_user_id = None
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
}
def __init__(self):
self.redis = Planet.redis
self.postgres = Planet.postgres
self.__get_my_info()
logging.info('Init my hash : %s and my user id : %s', Planet.my_hash,
Planet.my_user_id)
def handler(self, sql, params):
""" 处理数据
:param sql:
:param params:
:return:
"""
return self.postgres.handler(sql, params)
def __get_my_info(self):
"""获取个人info"""
key = 'planet:my:token'
Planet.headers['Authorization'] = "token " + self.redis.get(key)
api = 'https://www.quanquanyuanyuan.cn/huodong/dog/api/my-dog-hash'
resp = requests.post(api, json={}, headers=Planet.headers).json()
Planet.my_hash = resp['uid_hash']
Planet.my_user_id = resp['user_id']
def __init__(self):
# PostgreSQL
host = 'localhost'
port = 12432
db_name = 'scrapy'
username = db_name
password = db_name
self.postgres = pgs.Pgs(host=host,
port=port,
db_name=db_name,
user=username,
password=password)
# Redis
self.redis = rds.Rds(host=host, port=12379, db=1,
password='******').redis_cli
import requests
from utils import pgs
nearjob = 'nearjob'
postgres = pgs.Pgs(host='localhost', port=12432, db_name=nearjob, user=nearjob, password=nearjob)
def get_city():
"""获取Boss直聘城市数据"""
api = 'https://www.zhipin.com/common/data/city.json'
resp = requests.get(api).json()
code = resp.get('rescode')
if code == 1: # 请求成功
data = resp['data']
# locationCity cityList
hot_city_list = data['hotCityList']
for hot_city in hot_city_list:
code = hot_city['code']
name = hot_city['name']
def get_position():
"""获取Boss直聘岗位数据"""
api = 'https://www.zhipin.com/common/data/position.json'
resp = requests.get(api).json()
code = resp.get('rescode')
if code == 1:
data_list = resp['data']
for data in data_list:
def postgres():
return pgs.Pgs(host=host,
port=12432,
db_name=near_job,
user=near_job,
password=near_job)
class Planet(object):
log.Logger()
# Postgres配置
host = 'localhost'
pgs_port = 12432
pgs_db_name = 'planet'
pgs_user = pgs_db_name
pgs_pwd = pgs_db_name
postgres = pgs.Pgs(host=host,
port=pgs_port,
db_name=pgs_db_name,
user=pgs_user,
password=pgs_pwd,
singleton=False)
# Redis配置
rds_port = 12379
rds_pwd = 'redis6379'
redis = rds.Rds(host=host, port=rds_port, password=rds_pwd).redis_cli
my_hash = None
my_user_id = None
headers = {}
def __init__(self):
self.redis = Planet.redis
self.postgres = Planet.postgres
self.__get_my_hash()
self.__get_my_token()
self.__get_my_user_id()
logging.info('Init my hash : %s and my user id : %s', Planet.my_hash,
Planet.my_user_id)
def handler(self, sql, params):
""" 处理数据
:param sql:
:param params:
:return:
"""
return self.postgres.handler(sql, params)
def __get_my_user_id(self):
"""获取userId值"""
if not Planet.my_user_id:
key = 'planet:my:uid'
Planet.my_user_id = self.redis.get(key)
if not Planet.my_user_id:
api = 'https://www.quanquanyuanyuan.cn/huodong/dog/api/my-status'
resp = requests.post(api, json={},
headers=Planet.headers).json()
Planet.my_user_id = resp['notification_settings'][0]['user_id']
self.redis.set(key, Planet.my_user_id)
def __get_my_hash(self):
"""获取hash值"""
if not Planet.my_hash:
key = 'planet:my:hash'
Planet.my_hash = self.redis.get(key)
if not Planet.my_hash:
api = 'https://www.quanquanyuanyuan.cn/huodong/dog/api/my-dog-hash'
resp = requests.post(api, json={},
headers=Planet.headers).json()
Planet.my_hash = resp['uid_hash']
self.redis.set(key, Planet.my_hash)
def __get_my_token(self):
"""获取token值"""
key = 'planet:my:token'
Planet.headers['Authorization'] = self.redis.get(key)
import time
import json
import random
import urllib3
import requests
from bs4 import BeautifulSoup
from datetime import datetime
from utils import pgs, es
urllib3.disable_warnings()
wx_mps = 'wxmps' # 这里数据库、用户、密码一致(需替换成实际的)
postgres = pgs.Pgs(host='localhost',
port='12432',
db_name=wx_mps,
user=wx_mps,
password=wx_mps)
elastic = es.Es(host='localhost', port=12900, index='mp', doc='article')
def load():
for i in [13, 14, 25, 3]:
sql = "select id,mps_biz,last_msg_id,app_msg_token,pass_ticket,wap_sid2 from " \
"tb_mps where id = {} and show = True".format(i)
result = postgres.fetch_all(sql)
for r in result:
r_id, r_mps_biz, r_last_msg_id, r_app_msg_token, r_pass_ticket, r_wap_sid2 = r[0], r[1], r[2], \
r[3], r[4], r[5]
r_cookie = 'wxuin=1604513290; version=62060619; lang=zh_TW; pass_ticket={}; wap_sid2={}'.format(
r_pass_ticket, r_wap_sid2)