def getCookie(key, mode=1):
if mode == 0:
from middles.middleAssist import redisAsisst
Rs = redisAsisst.imredis().connection()
elif mode == 1:
from middles.middleAssist import ssdbAssist
Rs = ssdbAssist.SSDBsession().connect()
else:
raise ValueError("mode is incorrect! it must be 1 or 0")
tt = Rs.hget("tmpaddress:cookie", key)
print("getCookie:", tt)
return tt
def __init__(self):
super(Amap, self).__init__()
self.seed_url = "http://report.amap.com/ajax/getCityInfo.do?"
self.start_urls = (
"http://report.amap.com/ajax/districtRank.do?linksType={}&cityCode={}",
"http://report.amap.com/ajax/getCityRank.do",
"http://report.amap.com/ajax/cityDailyQuarterly.do?cityCode={}&year={}&quarter={}",
"http://report.amap.com/ajax/cityMergedHourly.do?cityCode={}",
"http://report.amap.com/ajax/congest/getHubs.do?linksType={}&prime=false&trafficid=&weekRadio=true",
"http://report.amap.com/ajax/congest/getCongestRank.do?city={}&prime=false&trafficid="
)
# self.Logger = logAsisst.imLog(sys.argv[1])()
self.seed_key = "tmp:amap:CityCode"
print(self.count)
self.Rconn = redisAsisst.imredis().connection()
self.SDBconn = ssdbAssist.SSDBsession().connect()
def RoboEasyLogin(key,mode=1):
if mode == 0:
from middles.middleAssist import redisAsisst
Rs = redisAsisst.imredis().connection()
userinfo = eval(Rs.hget("robo:accountInfo", key))
elif mode == 1:
from middles.middleAssist import ssdbAssist
Rs = ssdbAssist.SSDBsession().connect()
userinfo = eval(Rs.hget("robo:accountInfo", key))
else:
raise ValueError("mode is incorrect! it must be 1 or 0")
user_data = {
"username": userinfo["account"][0],
"password": userinfo["account"][1],
"rememberMe": "false",
"app": "cloud"}
cookies = userinfo["cookie"]
loginurl = "https://app.datayes.com/server/usermaster/authenticate/v1.json"
ck = login(url=loginurl, cookies=cookies, user_data=user_data)
ck = cookiesSmooth(ck)
Rs.hset("tmpaddress:cookie", key, ck)
def giveCookie(method="set", **kwargs):
"""
:param method: set/get
:return:
"""
import requests
from middles.middleAssist import ssdbAssist
SDBconn = ssdbAssist.SSDBsession().connect()
url = "http://www.macrodb.com/ztxh/xh3_pro.asp?ways={}"
if method == "set":
print("Getting cookie...")
try:
tt = "http://www.macrodb.com:8000/data_m/m_hg1/frmdiv.asp?ways={}&dr=m_hg1"
url = tt.format(kwargs["code"])
except:
url = url.format("TCIK")
print("SetCookie >>> ", url)
ip = {"https": ipProxy.ipRandom["https"]}
req = requests.get(url=url,
proxies=ip,
headers={"user-agent": userAgent.user_agent})
print("Getting cookie finish!")
cookies = req.cookies.get_dict()
ck = ""
for cookie in cookies:
tmp = "{}={}; ".format(cookie, cookies[cookie])
ck += tmp
print("setting... ", ck)
SDBconn.set("ZhongTou:Cookie", ck)
return ck
elif method == "get":
ck = SDBconn.get("ZhongTou:Cookie")
print("getting... ", ck)
return ck
else:
raise ValueError("no such method values! It must be 'set' or 'get'.")
def __init__(self):
super(Yiche, self).__init__()
self.seed_urls = (
"http://index.bitauto.com/yicheindexpublic/rank/car-level",
"http://index.bitauto.com/yicheindexpublic/data/last-date",
"http://index.bitauto.com/ai/v4/searchparam/getCompeteCarsPublic")
self.allow_domains = ["http://index.bitauto.com/yicheindexpublic"]
self.obj_urls = (
"/sale/saleTrend",
"/sale/saleCountryByMonthLine",
"/sale/saleLevelBar",
"/sale/saleDynamicBar",
"/sale/saleMakeBar",
"/sale/saleLevelBubule",
"/sale/saleCountryPie",
"/indextrend",
"/praisetrend",
"/rank/list",
)
self._seed_key = "tmp:yiche:CarModel"
self._Rconn = redisAsisst.imredis().connection()
self._SDBconn = ssdbAssist.SSDBsession().connect()
self._Mconn = mysqlAssist.immysql()
sys.setdefaultencoding('utf-8')
import os
currentUrl = os.path.dirname(__file__)
parentUrl = os.path.abspath(os.path.join(currentUrl, os.pardir))
sys.path.append(parentUrl)
import pinyin
from middles.middleAssist import ssdbAssist
from middles.middleAssist import redisAsisst
from middles.middleWare import fakeQueue
import hashlib
SSDB_DEFAULT_KEY = "s_o_d_i_{}"
seriesId = 0
treeId = 1
_SSDB = ssdbAssist.SSDBsession()
_Rconn = redisAsisst.imredis().connection()
def str2md5(s, p=8):
md = hashlib.md5(s)
a = md.hexdigest()[:p]
return a
NodeDATAQueue="tmpCollectDATAqueue"
class uploadMenu(object):
def __init__(self,conn, plat, channel, prefix="YC"):
self.conn = conn
self.plat = plat
obj_dict = {
1: "单日票房:趋势分析",
0: "单日票房:单日票房",
}
channel_dict = {
0: '单日票房',
1: '单日趋势分析',
2: '单周票房',
3: '周末票房',
4: '单月票房',
5: '年度票房',
6: '年度首周票房',
7: '影院票房',
}
ssdb_conn = ssdbAssist.SSDBsession().connect()
mysql_conn = mysqlAssist.immysql()
redis_conn = redisAsisst.imredis().connection()
class YienTubes(BaseTubes):
def __init__(self):
super(YienTubes, self).__init__()
self.obj = Yien()
self.plat_id = 8
self.channel = ("yiendata", "艺恩数据")
def tubes_detail(self, code):
cod = code
gt = self.obj.parse(cod)