def __init__(self, platid=None, taskid=None, objid=None,**kwargs):
BaseTubes.__init__(self, platid = platid, taskid= taskid,
objid = objid)
self.obj = Robo(hkey=kwargs["hkey"])
if kwargs['sqlopen'] :
self.sql = mysqlAssist.immysql()
self.redis = redisAsisst.imredis().connection()
self.channelCode = None
#self.redis = redisAsisst.imredis().connection()
self.Logger = logAsisst.imLog(sys.argv[1])()
self.feature_code = "seen:code"
def getCookie(key, mode=1):
if mode == 0:
from middles.middleAssist import redisAsisst
Rs = redisAsisst.imredis().connection()
elif mode == 1:
from middles.middleAssist import ssdbAssist
Rs = ssdbAssist.SSDBsession().connect()
else:
raise ValueError("mode is incorrect! it must be 1 or 0")
tt = Rs.hget("tmpaddress:cookie", key)
print("getCookie:", tt)
return tt
def parse_item(self, response):
from bs4 import BeautifulSoup
import urlparse
from middles.middleAssist import redisAsisst
Rconn = redisAsisst.imredis().connection()
compare_url = 'http://www.macrodb.com:8000/data_m/xx/xx'
soup = BeautifulSoup(response, "html.parser")
tags = soup.select("frame")
fake_url = tags[-1]['src']
t = urlparse.urljoin(compare_url, fake_url)
gen = self.parse_detail(url=t, channelname=self.channelname[0])
for i in gen:
Rconn.sadd("ZT:obj", i)
def __init__(self):
super(Amap, self).__init__()
self.seed_url = "http://report.amap.com/ajax/getCityInfo.do?"
self.start_urls = (
"http://report.amap.com/ajax/districtRank.do?linksType={}&cityCode={}",
"http://report.amap.com/ajax/getCityRank.do",
"http://report.amap.com/ajax/cityDailyQuarterly.do?cityCode={}&year={}&quarter={}",
"http://report.amap.com/ajax/cityMergedHourly.do?cityCode={}",
"http://report.amap.com/ajax/congest/getHubs.do?linksType={}&prime=false&trafficid=&weekRadio=true",
"http://report.amap.com/ajax/congest/getCongestRank.do?city={}&prime=false&trafficid="
)
# self.Logger = logAsisst.imLog(sys.argv[1])()
self.seed_key = "tmp:amap:CityCode"
print(self.count)
self.Rconn = redisAsisst.imredis().connection()
self.SDBconn = ssdbAssist.SSDBsession().connect()
def RoboEasyLogin(key,mode=1):
if mode == 0:
from middles.middleAssist import redisAsisst
Rs = redisAsisst.imredis().connection()
userinfo = eval(Rs.hget("robo:accountInfo", key))
elif mode == 1:
from middles.middleAssist import ssdbAssist
Rs = ssdbAssist.SSDBsession().connect()
userinfo = eval(Rs.hget("robo:accountInfo", key))
else:
raise ValueError("mode is incorrect! it must be 1 or 0")
user_data = {
"username": userinfo["account"][0],
"password": userinfo["account"][1],
"rememberMe": "false",
"app": "cloud"}
cookies = userinfo["cookie"]
loginurl = "https://app.datayes.com/server/usermaster/authenticate/v1.json"
ck = login(url=loginurl, cookies=cookies, user_data=user_data)
ck = cookiesSmooth(ck)
Rs.hset("tmpaddress:cookie", key, ck)
def __init__(self):
super(Yiche, self).__init__()
self.seed_urls = (
"http://index.bitauto.com/yicheindexpublic/rank/car-level",
"http://index.bitauto.com/yicheindexpublic/data/last-date",
"http://index.bitauto.com/ai/v4/searchparam/getCompeteCarsPublic")
self.allow_domains = ["http://index.bitauto.com/yicheindexpublic"]
self.obj_urls = (
"/sale/saleTrend",
"/sale/saleCountryByMonthLine",
"/sale/saleLevelBar",
"/sale/saleDynamicBar",
"/sale/saleMakeBar",
"/sale/saleLevelBubule",
"/sale/saleCountryPie",
"/indextrend",
"/praisetrend",
"/rank/list",
)
self._seed_key = "tmp:yiche:CarModel"
self._Rconn = redisAsisst.imredis().connection()
self._SDBconn = ssdbAssist.SSDBsession().connect()
self._Mconn = mysqlAssist.immysql()
import os
currentUrl = os.path.dirname(__file__)
parentUrl = os.path.abspath(os.path.join(currentUrl, os.pardir))
sys.path.append(parentUrl)
import pinyin
from middles.middleAssist import ssdbAssist
from middles.middleAssist import redisAsisst
from middles.middleWare import fakeQueue
import hashlib
SSDB_DEFAULT_KEY = "s_o_d_i_{}"
seriesId = 0
treeId = 1
_SSDB = ssdbAssist.SSDBsession()
_Rconn = redisAsisst.imredis().connection()
def str2md5(s, p=8):
md = hashlib.md5(s)
a = md.hexdigest()[:p]
return a
NodeDATAQueue="tmpCollectDATAqueue"
class uploadMenu(object):
def __init__(self,conn, plat, channel, prefix="YC"):
self.conn = conn
self.plat = plat
self.channel = channel
def __init__(self, platid=None, taskid=None, objid=None, **kwargs):
global logname
BaseTubes.__init__(self, platid=platid, taskid=taskid, objid=objid)
self._Rconn = redisAsisst.imredis().connection()
self._Mconn = mysqlAssist.immysql()