def parse_user(self, response):
'''
因为返回的是json格式的数据，所以这里直接通过json.loads获取结果
:param response:
:return:
'''
result = json.loads(response.text)
item = UserItem()
# 这里循环判断获取的字段是否在自己定义的字段中，然后进行赋值
for field in item.fields:
if field in result.keys():
item[field] = result.get(field)
# 这里在返回item的同时返回Request请求，继续递归拿关注用户信息的用户获取他们的关注列表
print(item)
yield item
yield scrapy.Request(self.follows_url.format(
user=result.get("url_token"),
include=self.follows_query,
offset=0,
limit=20),
headers=self.headers,
callback=self.parse_follows)
yield scrapy.Request(self.followers_url.format(
user=result.get("url_token"),
include=self.followers_query,
offset=0,
limit=20),
headers=self.headers,
callback=self.parse_followers)
def user_parse(self, response):
result = json.loads(response.text)
pprint(result)
item = UserItem()
for field in item.fields:
if field in result.keys():
item[field] = result.get(field)
yield item
yield Request(self.url_follow.format(user=result.get('url_token'),
include=self.follow_include,
offset=0),
callback=self.follow_parse)
def parse_user(self, response):
result = json.loads(response.text)
item = UserItem()
for field in item.fields:
if field in result.keys():
item[field] = result.get(field)
yield item
yield Request(
self.follows_url.format(user=result.get('url_token'),
include=self.follows_query,
limit=20,
offset=0), self.parse_follows)
yield Request(
self.followers_url.format(user=result.get('url_token'),
include=self.followers_query,
limit=20,
offset=0), self.parse_followers)
def parse_user(self, response):
"""获取用户信息，并分别调用方法获取他们的关注与粉丝"""
result = json.loads(response.text)
item = UserItem()
for field in item.fields:
if field in result.keys():
item[field] = result.get(field)
yield item
# 对获取到的用户，再获取他们的粉丝列表
yield Request(
self.followers_url.format(user=result.get('url_token'),
include=self.followers_query,
offset=0,
limit=20), self.parse_followers)
# 获取他们的关注列表
yield Request(
self.follows_url.format(user=result.get('url_token'),
include=self.follows_query,
offset=0,
limit=20), self.parse_follows)