def close_spider(self, spider=None):
CommonScrapyPipelineClass.log_close_spider(spider_obj=spider)
if spider.xy_seen_updated_bool:
log_file_path = os.path.join(spider.log_dir,
spider.xy_response_log_file_name)
bak_file_name = spider.xy_response_log_file_name.replace(
".log", ".bak")
bak_file_path = os.path.join(spider.log_dir, bak_file_name)
if os.path.isfile(log_file_path):
shutil.copyfile(log_file_path, bak_file_path)
try:
with open(log_file_path, "w", encoding="utf-8") as xy_log_file:
for index, center_xy_str in enumerate(spider.xy_seen_dict):
item = spider.xy_seen_dict[center_xy_str]
xy_log_file.write(f"{center_xy_str},{item}\n")
except Exception as ex:
spider.logger.error(
f"cannot write historical xy_log_file ({log_file_path}). Exception = {ex}"
)
def close_spider(self, spider=None):
CommonScrapyPipelineClass.log_close_spider(spider_obj=spider)