async def process(self, response: Response):
status = response.json().get('status')
if status:
data = response.json().get('data')
for k in self.keys():
if k in data:
self[k] = data[k]
with open('user_' + str(self['mid']) + '.txt', 'w') as f:
f.write(self.dumps() + '\n')
async def videoinfo(response: Response):
if response.status_code == 200 and response.json()['status']:
params = re.findall('mid=(.*?)&pagesize=30&tid=0&page=(.*?)&keyword',
response.url)[0]
yield Request(url=video_url.format(params[0],
int(params[1]) + 1),
callback=videoinfo)
data = response.json()['data']['vlist']
for d in data:
db['video'].update_one({'aid': d['aid']}, {'$set': d}, True)
yield Request(url=av_url.format(aid=d['aid']), callback=AVPage)
async def process(self, response: Response):
status = response.json().get('status')
if status:
data = response.json().get('data')
pattern = 'mid=(\d+?)&pagesize=30&tid=0&page=(\d+?)&keyword=&order=pubdate'
patn = re.findall(pattern, response.url)[0]
print(patn)
yield Request(url=videos_url.format(mid=patn[0],
page=int(patn[1]) + 1),
callback=VideoInfo),
for v in data['vlist']:
yield Request(url=av_url.format(aid=v.get('aid')), callback=AV)
async def process(self, response: Response):
file_name = response.current_request.meta.get('name')
if file_name and response.status_code == 200:
with open(file_name + '.mp4', 'wb') as f:
for content in response.iter_content(chunk_size=512):
f.write(content)
f.flush()
async def download(response: Response):
if response.status_code == 200:
chunk_size = 1024
name = response.current_request.meta.get('name')
with open(name, 'wb') as f:
for content in response.iter_content(chunk_size=chunk_size):
f.write(content)
f.flush()
async def process(self, response: Response):
data = response.json()['data']
self['score'] = str(data['rate'])
self['count'] = str(data['userCount'])
self['id'] = response.current_request.meta.get('bookid')
# print(response.current_request.meta)
data = self.json()
print(data)
db['qidian'].update_one({'id': self['id']},
{"$set": dict(self.json())}, True)
async def process(self, response: Response):
if response.status_code == 200:
result = response.json()
if result['status']:
for key in self.keys():
if key in result['data']:
self[key] = result['data'][key]
self['level'] = result['data']['level_info']['current_level']
db['user'].update_one({'mid': self['mid']},
{"$set": self.json()}, True)
yield Request(url=followed_url.format(
mid=result['data']['mid'], page=1),
callback=follow)
yield Request(url=follower_url.format(
mid=result['data']['mid'], page=1),
callback=follow)
async def process(self, response: Response):
for data in response.json():
yield Request(av_url.format(aid=data[1]), callback=AV)