def __init__(self, platid=None, taskid=None, objid=None, **kwargs):
global logname
BaseTubes.__init__(self, platid=platid, taskid=taskid, objid=objid)
self.obj = Yiche()
self._Mconn = mysqlAssist.immysql()
self.Logger = logAsisst.imLog(logname)()
def __init__(self, platid=None, taskid=None, objid=None):
BaseTubes.__init__(self, platid = platid, taskid= taskid,
objid = objid)
self.obj = Robo()
self.sql = mysqlAssist.immysql()
self.channelCode = None
self.redis = ""
self.Logger = logAsisst.imLog("RoboCrawls")()
self.pick_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
def __init__(self, open_sql=False):
super(AliInd, self).__init__()
self.start_urls = 'https://index.1688.com/alizs/market.htm?spm=a262ha.8884008.0.0.uCrTZL'
self.seed_url = 'https://index.1688.com/alizs/market.htm?userType=purchaser&cat='
del ipProxy.ipRandom['https']
self.proxies = ipProxy.ipRandom
self.headers = {"user-agent": userAgent.user_agent}
self.tmpurl = ""
if open_sql:
self._Mconn = mysqlAssist.immysql()
def __init__(self, platid=None, taskid=None, objid=None,**kwargs):
BaseTubes.__init__(self, platid = platid, taskid= taskid,
objid = objid)
self.obj = Robo(hkey=kwargs["hkey"])
if kwargs['sqlopen'] :
self.sql = mysqlAssist.immysql()
self.redis = redisAsisst.imredis().connection()
self.channelCode = None
#self.redis = redisAsisst.imredis().connection()
self.Logger = logAsisst.imLog(sys.argv[1])()
self.feature_code = "seen:code"
def __init__(self, platid=None, taskid=None, objid=None, **kwargs):
global logname
super(AliIndTubes, self).__init__()
BaseTubes.__init__(self, platid=platid, taskid=taskid, objid=objid)
self.obj = AliInd()
self._Mconn = mysqlAssist.immysql()
self.log = logAsisst.imLog(logname)()
self.ep = EasyUploadMenu.uploadMenu(plat=7,
channel="阿里指数",
conn=self.Conn(),
prefix="AL")
self.ep.setChannelCode(name="Alizs")
def tmp_crawl(code, objname):
sql = "select seed, seed_val from t_ext_seed_data_copy where platform='Alizs'" \
" and pseed=%s"%code
# p = AliIndTubes()
# sql_result = p.Conn().query(sql)
mconn = mysqlAssist.immysql()
sql_result = mconn.query(sql)
del mconn
p = AliIndTubes()
if sql_result:
gs = []
for s in sql_result:
name = "{}:{}".format(objname, s[1])
name = re.sub("^:", "", name)
p.tubes_detail(s[0], name)
tmp_crawl(s[0], name)
def __init__(self):
super(Yiche, self).__init__()
self.seed_urls = (
"http://index.bitauto.com/yicheindexpublic/rank/car-level",
"http://index.bitauto.com/yicheindexpublic/data/last-date",
"http://index.bitauto.com/ai/v4/searchparam/getCompeteCarsPublic")
self.allow_domains = ["http://index.bitauto.com/yicheindexpublic"]
self.obj_urls = (
"/sale/saleTrend",
"/sale/saleCountryByMonthLine",
"/sale/saleLevelBar",
"/sale/saleDynamicBar",
"/sale/saleMakeBar",
"/sale/saleLevelBubule",
"/sale/saleCountryPie",
"/indextrend",
"/praisetrend",
"/rank/list",
)
self._seed_key = "tmp:yiche:CarModel"
self._Rconn = redisAsisst.imredis().connection()
self._SDBconn = ssdbAssist.SSDBsession().connect()
self._Mconn = mysqlAssist.immysql()
def __init__(self, platid=None, taskid=None, objid=None, **kwargs):
global logname
BaseTubes.__init__(self, platid=platid, taskid=taskid, objid=objid)
self._Rconn = redisAsisst.imredis().connection()
self._Mconn = mysqlAssist.immysql()
1: "单日票房:趋势分析",
0: "单日票房:单日票房",
}
channel_dict = {
0: '单日票房',
1: '单日趋势分析',
2: '单周票房',
3: '周末票房',
4: '单月票房',
5: '年度票房',
6: '年度首周票房',
7: '影院票房',
}
ssdb_conn = ssdbAssist.SSDBsession().connect()
mysql_conn = mysqlAssist.immysql()
redis_conn = redisAsisst.imredis().connection()
class YienTubes(BaseTubes):
def __init__(self):
super(YienTubes, self).__init__()
self.obj = Yien()
self.plat_id = 8
self.channel = ("yiendata", "艺恩数据")
def tubes_detail(self, code):
cod = code
gt = self.obj.parse(cod)
for i in gt:
def GenSeed(code, name=None):
sql = "SELECT seed,seed_val FROM `t_ext_seed_data`" \
" WHERE `platform` = 'Alizs' AND `pseed` = '%s'"%code
conn = mysqlAssist.immysql()
for i in conn.query(sql):
yield i