def __init__(self,
url,
uuid,
upload_url,
callback,
check_video_url=None,
*args,
**kwargs):
super(BilibiliSpider, self).__init__(*args, **kwargs)
self.config = ConfigParser.ConfigParser()
self.config.read("config/config.ini")
self.uuid = uuid
self.upload_url = upload_url
self.callback = callback
self.check_video_url = check_video_url
# initialize db
with open("config/database.cnf") as f:
config = json.load(f)
db_cls = get_database(config.get("database_type", None))
self.db = db_cls(**config.get("database", {}))
self.start_urls.append(url)
#浏览器复制bilibili的cookie
self.header = {
'Cookie':
'pgv_pvi=1211209728; UM_distinctid=15b05e151043a-02ba96c782db3a-6a11157a-100200-15b05e1510774; fts=1490452304; sid=56ptt8q6; rpdid=owkliqwxxldoploikssww; finger=7360d3c2; buvid3=927004CE-F581-4EC0-91C4-5832AC9038D025906infoc; pgv_si=s5848608768; purl_token=bilibili_1496493155'
}
def __init__(self, url, uuid, upload_url, callback, check_video_url=None, *args, **kwargs):
super(EchoSpider, self).__init__(*args, **kwargs)
self.config = ConfigParser.ConfigParser()
self.config.read("config/config.ini")
self.uuid = uuid
self.upload_url = upload_url
self.callback = callback
self.check_video_url = check_video_url
# initialize db
with open("config/database.cnf") as f:
config = json.load(f)
db_cls = get_database(config.get("database_type", None))
self.db = db_cls(**config.get("database", {}))
self.start_urls.append(url)
print url
self.headers = {
'User-Agent': 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 6 Build/LYZ28E) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Mobile Safari/537.36',
'Host': 'www.app-echo.com',
'Cookie': 'PHPSESSID=4d8dosv6miubuh1igoc6ofdfu6; echo_language=0fa769e85f49c8f39f1a51b419d5ec98c7821fcdb7666236b7c498a20cee27fea%3A2%3A%7Bi%3A0%3Bs%3A13%3A%22echo_language%22%3Bi%3A1%3Bs%3A2%3A%22cn%22%3B%7D; _csrf=0f742eba9cca1b0785a3283f9e28b89cb4ae5a6ceeaf35d328ba62505bb8751ca%3A2%3A%7Bi%3A0%3Bs%3A5%3A%22_csrf%22%3Bi%3A1%3Bs%3A32%3A%22Nd4K1ScPvyylRf2f2iO2n3opdyrDvogR%22%3B%7D; view_statistics_type=event.music-festival; Hm_lvt_8c9a0b394fc1f4d9177f4869cfd72618=1484472213; Hm_lpvt_8c9a0b394fc1f4d9177f4869cfd72618=1484472213; Hm_lvt_46b3b8e7eb78200527b089c276c81a7e=1484472113; Hm_lpvt_46b3b8e7eb78200527b089c276c81a7e=1484479163; MP_LIST=',
'Connection': 'keep-alive',
'Cache-Control': 'max-age=0',
'Accept-Language': 'en,zh-CN;q=0.8,zh;q=0.6',
'Accept-Encoding': 'gzip, deflate, sdch',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
}
def __init__(self,
url,
uuid,
upload_url,
callback,
platform=11001,
check_video_url=None,
live_callback=None,
*args,
**kwargs):
super(CztvSpider, self).__init__(*args, **kwargs)
self.config = ConfigParser.ConfigParser()
self.config.read("config/config.ini")
self.uuid = uuid
self.upload_url = upload_url
self.callback = callback
self.check_video_url = check_video_url
self.live_callback = live_callback
# initialize db
with open("config/database.cnf") as f:
config = json.load(f)
db_cls = get_database(config.get("database_type", None))
self.db = db_cls(**config.get("database", {}))
self.platform = platform
self.start_urls.append(url)
def __init__(self,
url,
uuid,
upload_url,
callback,
check_video_url=None,
*args,
**kwargs):
super(VideoToutiaoSpider, self).__init__(*args, **kwargs)
self.config = ConfigParser.ConfigParser()
self.config.read("config/config.ini")
self.uuid = uuid
self.upload_url = upload_url
self.callback = callback
self.check_video_url = check_video_url
# initialize db
with open("config/database.cnf") as f:
config = json.load(f)
db_cls = get_database(config.get("database_type", None))
self.db = db_cls(**config.get("database", {}))
self.start_urls.append(url)
self.headers = {
'Host':
'www.toutiao.com',
'Cache-Control':
'max-age=0',
'Upgrade-Insecure-Requests':
1,
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'DNT':
1,
'Accept-Encoding':
'gzip, deflate, sdch',
'Accept-Language':
'zh-CN,zh;q=0.8,en;q=0.6,zh-TW;q=0.4',
'Cookie':
'uuid="w:26f6c318a3d54f359dde999ed9c05acc"; UM_distinctid=15ac5b41a4d64b-0dd4bf7186a738-1d3a6853-1fa400-15ac5b41a4e761; _ba=BA0.2-20170317-51d9e-fPGwZMlLK21rToVv50w2; csrftoken=92f3c0634079d6e3c96068bdcfa65ea3; tt_webid=54911862630; __tasessionId=iscec65nv1494484094230; CNZZDATA1259612802=1805146543-1486616419-%7C1494483501; _ga=GA1.2.1585727393.1486621132; _gid=GA1.2.190446546.1494484408'
}
def __init__(self,
url,
uuid,
upload_url,
callback,
check_video_url=None,
*args,
**kwargs):
super(AcfunSpider, self).__init__(*args, **kwargs)
self.config = ConfigParser.ConfigParser()
self.config.read("config/config.ini")
self.uuid = uuid
self.upload_url = upload_url
self.callback = callback
self.check_video_url = check_video_url
# initialize db
with open("config/database.cnf") as f:
config = json.load(f)
db_cls = get_database(config.get("database_type", None))
self.db = db_cls(**config.get("database", {}))
self.start_urls.append(url)
self.headers = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',
'Host':
'www.acfun.cn',
'Cookie':
'uuid=c9da88c1238512ed961a461e8bde57dd; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%221593df8a601f7-03485fb6b71f09-1d316f50-1fa400-1593df8a6022a3%22%7D; tma=191396026.46866448.1487671944652.1488970326344.1489398920581.4; tmd=22.191396026.46866448.1487671944652.; bfd_g=bffd842b2b484332000047ad00b6c72c55f7f17c; Hm_lvt_bc75b9260fe72ee13356c664daa5568c=1489547693; Hm_lpvt_bc75b9260fe72ee13356c664daa5568c=1489547822; Hm_lvt_bc75b9260fe72ee13356c664daa5568c=1489547693; Hm_lpvt_bc75b9260fe72ee13356c664daa5568c=1490673699; clientlanguage=zh_CN',
'Connection':
'keep-alive',
'Cache-Control':
'max-age=0',
'Accept-Language':
'zh-CN,zh;q=0.8,en;q=0.6,zh-TW;q=0.4',
'Accept-Encoding':
'gzip, deflate, sdch',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
}
def __init__(self,
url,
callback,
sub='vtt',
lang='en',
upload_url=None,
*args,
**kwargs):
super(SubtitleYoutubeSpider, self).__init__(*args, **kwargs)
self.start_urls.append(url)
self.config = ConfigParser.ConfigParser()
self.config.read("config/config.ini")
self.sub = sub
self.lang = lang
self.callback = callback
self.upload_url = upload_url
# initialize db
with open("config/database.cnf") as f:
config = json.load(f)
db_cls = get_database(config.get("database_type", None))
self.db = db_cls(**config.get("database", {}))
def __init__(self, url, uuid, upload_url, callback, check_video_url=None, live_callback=None, *args, **kwargs):
super(WeiboSpider, self).__init__(*args, **kwargs)
self.config = ConfigParser.ConfigParser()
self.config.read("config/config.ini")
self.uuid = uuid
self.upload_url = upload_url
self.callback = callback
self.check_video_url = check_video_url
self.live_callback = live_callback
# initialize db
with open("config/database.cnf") as f:
config = json.load(f)
db_cls = get_database(config.get("database_type", None))
self.db = db_cls(**config.get("database", {}))
self.start_urls.append(url)
self.username = '******'
self.password = '******'
self.prelogin = '******'
self.login_url = 'https://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.18)'
self.headers = {
'User-Agent': 'Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_2 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5'}
