def generate_filename_from_url(self, url="", file_type=""):
response_html = ""
filename = ""
filename_base = ""
folder = self.list_html_folder_name
now = datetime.datetime.now()
today = now.strftime('%Y%m%d')
url_fragments = url.split("/")
while '' in url_fragments:
url_fragments.remove('')
# Examples:
# http://www.dianping.com/chenzhou/ch10/g113
# http://www.dianping.com/shop/72457872
# http://www.dianping.com/shop/8910906/review_all/p624
if "list2" == file_type:
if 3 < len(url_fragments):
filename_base = f"{url_fragments[-3]}_{url_fragments[-2]}_{url_fragments[-1]}"
response_html = f"{filename_base}_{today}.html"
filename = response_html
elif "detailed" == file_type:
folder = self.detail_html_folder_name
if 3 < len(url_fragments) and "review_all" == url_fragments[-2]:
shop_id = CommonClass.find_digits_from_str(url_fragments[-3])
filename_base = f"shop_{shop_id}_{url_fragments[-1]}"
response_html = f"{filename_base}_{today}.html"
filename = response_html
elif 2 < len(url_fragments):
shop_id = CommonClass.find_digits_from_str(url_fragments[-1])
filename_base = f"shop_{shop_id}_p1"
response_html = f"{filename_base}_{today}.html"
filename = response_html
elif "css" == file_type:
# http://s3plus.meituan.net/v1/mss_0a06a471f9514fc79c981b5466f56b91/svgtextcss/a59454e0c1813952099c1e006c298195.css
folder = self.svg_text_css_folder_name
if 1 < len(url_fragments) and url_fragments[-1].endswith(".css"):
filename_base = url_fragments[-1].replace(".css", "")
response_html = url_fragments[-1]
filename = response_html
if response_html is None or 1 > len(response_html):
rand_int = random.randint(100000, 999999)
response_html = f"unknown{rand_int}_{today}.html"
self.logger.error(
f"File {response_html} is used to store html page crawled from {url}"
)
return response_html, folder, filename, filename_base
def get_page_from_url(self, url=""):
page_num = 0
url_obj = parse.urlparse(url)
if hasattr(url_obj, "path"):
url_list = url_obj.path.split("/")
for one in url_list:
if 0 == one.find("pn"):
page_num = CommonClass.find_digits_from_str(
string=one, return_all=False)
return int(page_num)
def find_more_house_ids(self, doc = ""):
house_id_list = []
counter = 0
index = 0
while True:
index = doc.find("data-hid", index)
if -1 == index:
break
sub_doc = doc[index+10:index+25]
house_id_list.append( CommonClass.find_digits_from_str( sub_doc ) )
index += 10
counter += 1
return house_id_list
def get_page_area_district_from_url(self, url_object=None):
"""
https://fs.58.com/shangpucz/
https://gz.58.com/shangpu/
https://gz.58.com/tianhe/shangpucz/pn3/?area=20_50
https://fs.58.com/foshan/shangpucz/pn2/ # foshan == 佛山周边，与禅城、高明、三水等同级
https://gz.58.com/shangpucz/pn3/
https://fs.58.com/shangpu/38143746902823x.shtml
"""
page = "1"
district = ""
shop_area = ""
detailed_page = False
if url_object is not None and hasattr(
url_object,
"netloc") and -1 < url_object.netloc.find("58.com"):
# parse query
has_shop_area = True
if not hasattr(url_object, "query") or 1 > len(url_object.query):
has_shop_area = False
if has_shop_area:
query_dict = parse.parse_qs(url_object.query)
if "area" in query_dict.keys() and isinstance(
query_dict["area"],
list) and 0 < len(query_dict["area"]):
shop_area = query_dict["area"][0]
# parse path
if hasattr(url_object, "path"):
url_list = url_object.path.split("/")
temp_list = []
for one in url_list:
if 0 < len(one) and -1 == one.find(
"shangpucz") and -1 == one.find(
"shangpu") and -1 == one.find("pn"):
temp_list.append(one)
elif -1 < one.find("pn"):
page = CommonClass.find_digits_from_str(
string=one, return_all=False)
elif -1 < one.find(".shtml"):
detailed_page = True
if not detailed_page and 1 == len(temp_list):
district = temp_list[0]
if detailed_page:
page = "0"
return (page, district, shop_area)
def read_json_and_parse(self, response):
file_list = os.listdir(self.json_dir)
# route0___0___20190615_234522.json
for one_file in file_list:
temp_list = one_file.split("___")
preset_route = 0
now = ""
if 2 < len(temp_list):
preset_route = temp_list[0]
preset_route = preset_route.lstrip("route")
preset_route = CommonClass.find_digits_from_str(
string=preset_route, return_all=False)
preset_route = int(preset_route)
now = temp_list[2]
now = now.rstrip(".json")
url = self.get_url_according_to_preset_route(
preset_route=preset_route)
json_file_path = os.path.join(self.json_dir, one_file)
if os.path.isfile(json_file_path):
try:
doc = None
with open(json_file_path, "rb") as f:
doc = f.read().decode("utf-8", "ignore")
if doc is None:
self.logger.error(
f"Error: cannot read html file {json_file_path}."
)
continue
text_dict = self.extract_text_dict_from_response_body(
body=doc, preset_route=preset_route, now=now)
if 0 < len(text_dict):
json_selector = Selector(text=doc, type=None)
loader = ItemLoader(item=DirectionbaiduItem(),
selector=json_selector)
loader = self.load_items_into_loader(
loader=loader,
text=text_dict,
url=url,
now=now)
yield loader.load_item()
except Exception as ex:
self.logger.error(
f"Inside Method {sys._getframe().f_code.co_name} of Class {self.__class__.__name__}, error happened during loading ItemLoader. Exception = {ex}"
)
def extract_all_detailed_html_links(self, string = ""):
house_id_list = []
if 1 > len( string ):
return house_id_list
doc = string.decode('utf-8')
end_string = '";var search_result_list_num ='
end_pos = len( doc )
if -1 < doc.find( end_string ):
end_pos = doc.find( end_string )
doc = doc[ len('var search_result = " '):end_pos ]
doc = '<!DOCTYPE html><html><head lang="zh-cn"><title>腾讯房产列表</title></head><body>' + f"{doc}</body></html>"
response = Selector( text=doc, type="html" )
house_id_list = response.xpath("//div/@data-hid").extract()
if 10 > len( house_id_list ):
house_id_list = self.find_more_house_ids( doc = doc )
else:
temp_list = []
for one_id in house_id_list:
temp_list.append( CommonClass.find_digits_from_str( one_id ) )
house_id_list = temp_list
return house_id_list
