def _deleted_spider(self, file_path, spider_data, templates): """ Add information about a deleted spider. """ spider = self._spider_name(file_path) file_path = self._spider_path(file_path) added = {file_path} added.update(set(templates.get(spider, []))) if self.ignore_deleted: return None, None, added spider_content = json.dumps(spider_data, sort_keys=True, indent=4) return file_path, spider_content, added
def _add_legacy_spider(self, file_path, templates, extractors): """ Build a legacy spider and add all templates to a single spider object """ spider = self._spider_name(file_path) file_path = self._spider_path(file_path) spider_data = self.read_file(file_path, deserialize=True) if spider_data.get('deleted'): return file_path, spider_data, {file_path} spider_data.pop('template_names', None) spider_templates = templates.get(spider, []) templates, added = self._spider_templates(spider_templates, extractors) added.add(file_path) spider_data['templates'] = templates return file_path, spider_data, added
def _add_legacy_spider(self, file_path, templates, extractors): """ Build a legacy spider and add all templates to a single spider object """ spider = self._spider_name(file_path) file_path = self._spider_path(file_path) spider_data = self.read_file(file_path, deserialize=True) if spider_data.get('deleted'): return file_path, spider_data, {file_path} names = set(spider_data.pop('template_names', [])) spider_templates = [tp for tp in templates.get(spider, []) if self._name(tp) in names] loaded_templates, added = self._spider_templates(spider_templates, extractors) added.add(file_path) spider_data['templates'] = loaded_templates return file_path, spider_data, added