def _spider_templates(self, spider_templates, extractors):
"""
Find all templates for a legacy spider and combine them into a single
list.
"""
templates, added = [], set()
for template_path in spider_templates:
added.add(template_path)
existing = {}
template = self.read_file(template_path, deserialize=True)
if template is None:
continue
if template.get('version', '') >= '0.13.0':
# Update `annotated_body`
annotations = template['plugins']['annotations-plugin']
apply_annotations(annotations, template)
template_extractors = template.get('extractors', {})
try:
for field, eids in template_extractors.items():
existing[field] = [eid for eid in eids
if eid in extractors]
template['extractors'] = existing
templates.append(template)
except AttributeError:
template['extractors'] = {}
return templates, added
def _spider_templates(self, spider_templates, extractors):
"""
Find all templates for a legacy spider and combine them into a single
list.
"""
templates, added = [], set()
for template_path in spider_templates:
added.add(template_path)
existing = {}
template = self.read_file(template_path, deserialize=True)
if template is None:
continue
if template.get('version', '') >= '0.13.0':
# Update `annotated_body`
annotations = template['plugins']['annotations-plugin']
apply_annotations(annotations, template)
template_extractors = template.get('extractors', {})
try:
for field, eids in template_extractors.items():
existing[field] = [
eid for eid in eids if eid in extractors
]
template['extractors'] = existing
templates.append(template)
except AttributeError:
template['extractors'] = {}
return templates, added
def _spider_templates(self, spider_templates, extractors):
"""
Find all templates for a legacy spider and combine them into a single
list.
"""
templates, added = [], set()
for template_path in spider_templates:
added.add(template_path)
existing = {}
template = self.read_file(template_path, deserialize=True)
if template is None:
continue
template_extractors = template.get('extractors', {})
for field, eids in template_extractors.items():
existing[field] = [eid for eid in eids if eid in extractors]
template['extractors'] = existing
templates.append(template)
return templates, added
def _spider_templates(self, spider_templates, extractors):
"""
Find all templates for a legacy spider and combine them into a single
list.
"""
templates, added = [], set()
for template_path in spider_templates:
added.add(template_path)
existing = {}
template = self.read_file(template_path, deserialize=True)
if template is None:
continue
template_extractors = template.get('extractors', {})
for field, eids in template_extractors.items():
existing[field] = [eid for eid in eids
if eid in extractors]
template['extractors'] = existing
templates.append(template)
return templates, added
