def infoParse(self,response):
request = checkTimeError(response)
if request:
yield request
return False
__ = S.replace_invalid_html_char(response.text)
response = response.replace(body = __)
# print(re.findall('(?=投[^>]*?资[^>]*?方).*?<a\s*?href=\".*?show(\d+)\/',response.text,re.S))
InfoConfigs = choice(response.url,contentsConfigs)
items = self.configParse(InfoConfigs,response,response)
for item in items.__iter__():
# yield item
print(item)
def configParse(self, configs, _response, response=None):
item = SzseItem()
if isinstance(configs, dict):
configs = [configs]
for _configs in configs:
#迭代可能多个的configs
if _configs.get('flag') is None:
_configs['flag'] = True
if _configs['list']['v'] and _configs['flag']:
res = S.select_content(_response, _configs['list'])
elif isinstance(_response, list):
res = _response
else:
#list(response) ----让response可迭代
res = [_response]
if res:
for _res in res:
#初始化result
result = dict()
#遍历每个字段提取
for config in _configs['data']:
k = config['En']
result[k] = S.select_content(_res, config, response)
result[k] = S.replace_invalid_char(result[k])
result[k] = S.replace_invalid_html_char(result[k])
if _configs.get('clear'):
for config in _configs['clear']:
k = config['En']
result[k] = S.select_content(
result[k], config, response)
item['result'] = result
item['keys'] = _configs['list']['keys']
item['db'] = _configs['list']['db']
item['conn'] = _configs['list'].get('conn')
#传递pipelin处理item字段
if result[_configs['list']['check']]:
yield item