def get_csv_categories(self):
while True:
try:
html_analyse = HtmlAnalyse(
"http://china.rs-online.com/web/c/pcb-prototyping/pcb-cleaning/",
proxy=self.proxy_ip)
bs_content = html_analyse.get_bs_contents()
break
except Exception as e:
print(sys._getframe().f_code.co_name, e)
self.proxy_pool.remove(self.proxy_ip)
self.proxy_ip = self.proxy_pool.get()
first_categories = bs_content.find_all(
name="div", attrs={"class": "horizontalMenu sectionUp"})
third_categories = []
for first_category in first_categories:
first_category_name = first_category.span.text
ul_tags = first_category.find_all(name="ul",
attrs={"class": "column1"})
for ul_tag in ul_tags:
li_tags = ul_tag.find_all(name="li")
for li_tag in li_tags:
second_category_url = Rs_Pre_Url + li_tag.a.get("href")
second_category_name = li_tag.a.text.replace(
li_tag.a.span.text, "").strip()
while True:
try:
html_analyse = HtmlAnalyse(second_category_url,
proxy=self.proxy_ip)
bs_content = html_analyse.get_bs_contents()
ul_tag = bs_content.find(
name="ul", attrs={"class": "brcategories"})
break
except Exception as e:
print(sys._getframe().f_code.co_name, e,
second_category_url)
self.proxy_pool.remove(self.proxy_ip)
self.proxy_ip = self.proxy_pool.get()
if ul_tag:
third_category_tags = ul_tag.find_all(
name="div", attrs={"class": "rsGARealEstate"})
for third_category_tag in third_category_tags:
third_category_name = third_category_tag.a.text
third_category_url = Rs_Pre_Url + third_category_tag.a.get(
"href")
third_category = (first_category_name,
second_category_name,
third_category_name,
third_category_url)
print(third_category)
third_categories.append(third_category)
else:
third_category = (first_category_name,
second_category_name,
second_category_name,
second_category_url)
print(third_category)
third_categories.append(third_category)
return third_categories
def get_suppliers(self):
def thread_go(page_url):
html_analyse = HtmlAnalyse(page_url)
while True:
try:
bs_content = html_analyse.get_bs_contents()
break
except Exception as e:
print(e)
company_tags = bs_content.find_all(name="a",
attrs={
"target": "_blank",
"href": re.compile(r"/\d+")
})
corporations = []
for company_tag in company_tags:
corporation = company_tag.text.strip()
corporation_dict = {
"corporation": corporation,
"province_url": city_url,
"page_url": page_url,
"状态": "未完成",
"from": "99114"
}
corporations.append(corporation)
col = self.db.All_Company_Name
col.insert(corporation_dict)
print(corporations)
return corporations
html_analyse = HtmlAnalyse("http://shop.99114.com/")
bs_content = html_analyse.get_bs_contents()
all_city_tags = bs_content.find_all(
name="a",
attrs={"href": re.compile(r"http://shop\.99114\.com/list/area")})
for city_tag in all_city_tags:
city_url = city_tag.get("href")
html_analyse = HtmlAnalyse(city_url)
bs_content = html_analyse.get_bs_contents()
page_tag = bs_content.find_all(
name="a", attrs={"href": re.compile(r"/list/area/")})[-2]
page_count = int(page_tag.text.replace(",", ""))
page_urls = map(
lambda page_num: city_url[:-1] + str(page_num) + ".html",
range(1, page_count + 1))
# for page_url in page_urls:
# thread_go(page_url)
threading_pool = ThreadingPool(12)
threading_pool.multi_process(thread_go, page_urls)
def get_categories(self):
main_url = "http://www.chip1stop.com/web/CHN/zh/dispClassSearchTop.do"
self.proxy_ip = self.proxy_pool.get()
while True:
try:
html_analsye = HtmlAnalyse(main_url, proxy=self.proxy_ip)
bs_content = html_analsye.get_bs_contents()
break
except Exception as e:
print(sys._getframe().f_code.co_name, e)
self.proxy_pool.remove(self.proxy_ip)
self.proxy_ip = self.proxy_pool.get()
dl_tags = bs_content.find_all(
name="dl", attrs={"class": "categoryListDl clearfix"})
second_categories = []
for dl_tag in dl_tags:
first_directory_name = dl_tag.dt.text
second_directory_tags = dl_tag.find_all(name="dd")
for second_directory_tag in second_directory_tags:
rough_second_directory_name = second_directory_tag.text
second_directory_name = re.match(
r"(.*?)\[", rough_second_directory_name).group(1).strip()
second_directory_url = "http://www.chip1stop.com/web/CHN/zh" + second_directory_tag.span.a.get(
"href")[1:]
second_directory = (first_directory_name,
second_directory_name,
second_directory_url)
second_categories.append(second_directory)
return second_categories
def thread_go(page_url):
html_analyse = HtmlAnalyse(page_url)
while True:
try:
bs_content = html_analyse.get_bs_contents()
break
except Exception as e:
print(e)
company_tags = bs_content.find_all(name="a",
attrs={
"target": "_blank",
"href": re.compile(r"/\d+")
})
corporations = []
for company_tag in company_tags:
corporation = company_tag.text.strip()
corporation_dict = {
"corporation": corporation,
"province_url": city_url,
"page_url": page_url,
"状态": "未完成",
"from": "99114"
}
corporations.append(corporation)
col = self.db.All_Company_Name
col.insert(corporation_dict)
print(corporations)
return corporations
def get_series_urls(self, list_url):
def get_pages_urls(url):
html_analyse = HtmlAnalyse(url)
bs_contents = html_analyse.get_bs_contents()
page_urls = []
page = len(
bs_contents.find_all(name="li", attrs={"class": "pager-item"
})) + 1
for i in range(page):
page_url = url.split('#')[0] + "?page=" + str(
i) + Third_Suffix_Url
page_urls.append(page_url)
return page_urls
series_urls = []
page_urls = get_pages_urls(list_url)
if page_urls is None:
return None
for page_url in page_urls:
html_analyse = HtmlAnalyse(page_url)
bs_contents = html_analyse.get_bs_contents()
tags = bs_contents.find_all(
name='tr',
attrs={"class": re.compile(u"ecatalog-series-table")})
for tag in tags:
try:
href = tag.find_all(name="td")[0].a.get("href")
m_url = Panasonic_Pre_Url + href + '&limit=100'
series_urls.append(m_url)
except Exception as e:
href = tag.find_all(name="td")[1].a.get("href")
m_url = Panasonic_Pre_Url + href + '&limit=100'
series_urls.append(m_url)
return series_urls
def thread_go(page_url):
html_analyse = HtmlAnalyse(page_url)
while True:
try:
bs_content = html_analyse.get_bs_contents()
break
except Exception as e:
print(e)
ul_tag = bs_content.find(name="div",
attrs={"class": "leftbox comlist"})
li_tags = ul_tag.find_all(name="li")
corporations = []
for li_tag in li_tags:
corporation = li_tag.text.strip()
corporation_dict = {
"corporation": corporation,
"province_url": province_url,
"page_url": page_url,
"状态": "未完成"
}
corporations.append(corporation)
col = self.db.All_Company_Name
col.insert(corporation_dict)
print(corporations)
return corporations
def get_supplier(self, url):
html_analyse = HtmlAnalyse(url)
bs_content = html_analyse.get_bs_contents()
ul_tag = bs_content.find(name="ul",
attrs={"class": "txl_content_con_L"})
supplier_name = ul_tag.h1.text.strip()
supplier_place = ul_tag.li.text.split("：", 2)[1].replace("\n",
" ").strip()
supplier_contact = ul_tag.find(name="li",
text=re.compile(r'联系人：')).text.split(
"：", 2)[1].strip()
supplier_fax = ul_tag.find(
name="li", text=re.compile(r'传真：')).text.split("：", 2)[1].strip()
supplier_phone = ul_tag.find(name="li",
text=re.compile(r'公司 联系电话：')).text.split(
"：", 2)[1].strip()
supplier_mobile = ul_tag.find(name="li",
text=re.compile(r'手机号码：')).text.split(
"：", 2)[1].strip()
supplier_address = ul_tag.find(name="li",
text=re.compile(r'联系地址：')).text.split(
"：", 2)[1].strip()
line = (supplier_name, supplier_place, supplier_contact, supplier_fax,
supplier_phone, supplier_mobile, supplier_address)
print(line)
return line
def get_products_list(self, series_url):
def get_pages_urls(url):
html_analyse = HtmlAnalyse(url, is_proxy=True)
bs_content = html_analyse.get_bs_contents()
page_tag = bs_content.find(name="a",
attrs={"title": "到最后一页"},
text="末页 »")
try:
rough_page = page_tag.get("href")
page = re.match(r"/ea/products/.*?page=(\d+)&reset=1",
rough_page).group(1)
except:
page = 0
page_urls = []
for i in range(int(page) + 1):
page_url = url + "&page=" + str(i)
page_urls.append(page_url)
return page_urls
product_lists = []
page_urls = get_pages_urls(series_url)
if page_urls is None:
return None
for page_url in page_urls[:]:
html_analyse = HtmlAnalyse(page_url)
bs_contents = html_analyse.get_bs_contents()
product_list = bs_contents.find_all(
name='tr', attrs={"class":
re.compile(u"(^odd$)|(^even$)")})[1:]
if not product_list:
continue
product_lists += product_list
return product_lists
def get_attach(detail_url):
html_analyse = HtmlAnalyse(detail_url)
bs_content = html_analyse.get_bs_contents()
rough_attach = bs_content.find(name="a", id="displayedPath")
try:
attach = ST_Pre_Url + rough_attach.get("href")
except:
attach = ""
return attach
def get_product_list(self):
html_analyse = HtmlAnalyse(self.url)
bs_contents = html_analyse.get_bs_contents()
rough_products = bs_contents.find_all(name="tr", attrs={"class": "products"})
products_urls = []
for rough_product in rough_products:
url = Infineon_Pre_Url + rough_product.td.a.get("href")
products_urls.append(url)
return products_urls
def get_pages_urls(url):
html_analyse = HtmlAnalyse(url)
bs_contents = html_analyse.get_bs_contents()
page_urls = []
page = len(bs_contents.find_all(name="li", attrs={"class": "pager-item"})) + 1
for i in range(page):
page_url = url.split('#')[0] + "?page=" + str(i) + Third_Suffix_Url
page_urls.append(page_url)
return page_urls
def get_img(url):
html_analyse = HtmlAnalyse(url)
bs_content = html_analyse.get_bs_contents()
rough_img = bs_content.find(name="img", id="productImageId")
try:
img = rough_img.get("src")
except:
print("未获取图片", url)
img = ''
return img
def get_supplier_urls(self, url):
html_analyse = HtmlAnalyse(url)
bs_content = html_analyse.get_bs_contents()
ul_tags = bs_content.find_all(name="ul",
attrs={"class": "sheng_weizhi_lb"})
urls = []
for ul_tag in ul_tags:
url = "http://book.youboy.com" + ul_tag.div.strong.a.get("href")
urls.append(url)
return urls
def get_code_urls(self, series_url):
def get_pages_urls(url):
html_analyse = HtmlAnalyse(url, is_proxy=True)
bs_content = html_analyse.get_bs_contents()
page_tag = bs_content.find(name="a",
attrs={"title": "到最后一页"},
text="末页 »")
try:
rough_page = page_tag.get("href")
page = re.match(r"/ea/products/.*?page=(\d+)&reset=1",
rough_page).group(1)
except:
page = 0
page_urls = []
for i in range(int(page) + 1):
page_url = url + "&page=" + str(i)
page_urls.append(page_url)
return page_urls
product_urls = []
page_urls = get_pages_urls(series_url)
if page_urls is None:
return None
for page_url in page_urls[:]:
html_analyse = HtmlAnalyse(page_url)
bs_contents = html_analyse.get_bs_contents()
lists = bs_contents.find_all(
name='tr', attrs={"class": re.compile(u"(^odd$)|(^even$)")})
if not lists:
continue
for list in lists[1:]:
try:
model = list.td.a
code = model.text
except:
break
# *******去重*******
orcl_con = OracleConnection()
cursor = orcl_con.conn.cursor()
cursor.execute(
"select cc_id from product$component_crawl where cc_code='{}'"
.format(code))
data = cursor.fetchone()
if data:
print("repeat")
continue
cursor.close()
orcl_con.conn.close()
# *******结束*******
href = model.get("href")
url = Panasonic_Pre_Url + href
product_urls.append(url)
return product_urls
def get_pdf(url):
html_analyse = HtmlAnalyse(url)
bs_content = html_analyse.get_bs_contents()
pdf = bs_content.find(
name="a",
attrs={"href": re.compile(r'/ac/c_download/.*?\.pdf')})
if pdf:
pdf_url = pdf.get("href") + "&via=ok"
else:
pdf_url = ''
return pdf_url
def get_product_url(self):
html_analyse = HtmlAnalyse(self.url)
bs_content = html_analyse.get_bs_contents()
rough_products = bs_content.find_all(name="div", attrs={"class": "section-devices"})
img = Atmel_Pre_Url + bs_content.find(name="img", attrs={"src": re.compile(r'/Images/.*?\.jpg')}).get("src")
imgs_urls = []
for rough_product in rough_products:
product_url = Atmel_Pre_Url + rough_product.a.get("href")
img_url = (img, product_url)
imgs_urls.append(img_url)
return imgs_urls
def belling(url):
html_analyse = HtmlAnalyse(url)
bs_content = html_analyse.get_bs_contents()
pdf_tags = bs_content.find_all(name="a",
attrs={"href": re.compile(r".*?\.pdf$")})
hrefs = []
for pdf_tag in pdf_tags:
href = pdf_tag.get("href")
print(href)
hrefs.append(href)
return hrefs
def get_pdf(url):
html_analyse = HtmlAnalyse(url)
bs_content = html_analyse.get_bs_contents()
pdf = bs_content.find(
name="a",
attrs={
"href":
re.compile(
r'/ac/c_download/control/relay/photomos/catalog/semi_cn_'
)
})
pdf_url = pdf.get("href") + "&via=ok"
return pdf_url
def get_pages_urls(url):
html_analyse = HtmlAnalyse(url, is_proxy=True)
bs_content = html_analyse.get_bs_contents()
page_tag = bs_content.find(name="a", attrs={"title": "到最后一页"}, text="末页 »")
try:
rough_page = page_tag.get("href")
page = re.match(r"/ea/products/.*?page=(\d+)&reset=1", rough_page).group(1)
except:
page = 0
page_urls = []
for i in range(int(page) + 1):
page_url = url + "&page=" + str(i)
page_urls.append(page_url)
return page_urls
def get_all_content(self):
many_contents = []
for series_url in self.series_urls:
if series_url == "http://device.panasonic.cn/ac/c/control/sensor/human/wl/number/index.jsp?c=search":
session = requests.session()
session.headers.update({
'Connection': 'keep-alive',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'zh-CN,zh;q=0.8',
'Accept-Encoding': 'gzip, deflate',
'Cache-Control': 'max-age=0',
'Content-Type': 'application/x-www-form-urlencoded',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.101 Safari/537.36',
'X-Requested-With': 'XMLHttpRequest',
'Host': 'device.panasonic.cn',
'Origin': 'http://device.panasonic.cn',
'Referer':
'http://device.panasonic.cn/ac/c/control/sensor/human/wl/number/index.jsp?c=search',
'Upgrade-Insecure-Requests': '1'
})
form = {
'pagecnt': 1,
'maxrows': 20,
'topage': 2,
'VAL_3_3286': '',
'VAL_3_3433': '',
'VAL_3_3287': '',
'VAL_3_3436': '',
'part_no': ''
}
content0 = session.get(series_url).text
bs_contents0 = BeautifulSoup(content0, "html.parser")
many_contents.append(bs_contents0)
content1 = session.post(
"http://device.panasonic.cn/ac/c/control/sensor/human/wl/number/index.jsp?c=move",
data=form).text
bs_contents1 = BeautifulSoup(content1, "html.parser")
many_contents.append(bs_contents1)
else:
html_analyse = HtmlAnalyse(series_url)
bs_contents = html_analyse.get_bs_contents()
many_contents.append(bs_contents)
return many_contents
def thread_go(self, parameters):
url = parameters
html_analyse = HtmlAnalyse(url)
bs_content = html_analyse.get_bs_contents()
tr_tags = bs_content.find(name="tbody").find_all(name="tr")
for tr_tag in tr_tags:
td_tags = tr_tag.find_all("td")
property_name = td_tags[0].text.strip()
value_tag = td_tags[1]
if property_name == "料号":
cc_code = value_tag.text.strip()
elif property_name == "品牌":
cc_brandname = value_tag.text.strip()
elif property_name == "规格书":
cc_attach = value_tag.get("href")
property_value = td_tags[1]
component = (cc_code, cc_brandname, cc_unit, cc_kiname, cc_url,
cc_attach, cc_img)
def get_suppliers(self):
def thread_go(page_url):
html_analyse = HtmlAnalyse(page_url)
while True:
try:
bs_content = html_analyse.get_bs_contents()
break
except Exception as e:
print(e)
ul_tag = bs_content.find(name="div",
attrs={"class": "leftbox comlist"})
li_tags = ul_tag.find_all(name="li")
corporations = []
for li_tag in li_tags:
corporation = li_tag.text.strip()
corporation_dict = {
"corporation": corporation,
"province_url": province_url,
"page_url": page_url,
"状态": "未完成"
}
corporations.append(corporation)
col = self.db.All_Company_Name
col.insert(corporation_dict)
print(corporations)
return corporations
for province_id in range(1, 36):
province_url = "http://www.soudh.com/province-" + str(
province_id) + ".html"
html_analyse = HtmlAnalyse(province_url)
bs_content = html_analyse.get_bs_contents()
page_tag = bs_content.find(name="span", text=re.compile(r'当前为'))
page_count = int(re.match(r'.*?共(\d+)页', page_tag.text).group(1))
page_urls = map(
lambda page_num: province_url[:-5] + "-" + str(page_num) +
".html", range(1, page_count + 1))
#
# for page_url in page_urls:
# thread_go(page_url)
threading_pool = ThreadingPool()
threading_pool.multi_thread(thread_go, page_urls)
def get_first_classes(
self, url="http://www.mlcc1.com/search_simplex.html?searchkey="):
html_analyse = HtmlAnalyse(url)
bs_content = html_analyse.get_bs_contents()
first_tag_names = bs_content.find_all(name="p",
attrs={"class": "down"})
first_class_contents = bs_content.find_all(
name="ul", attrs={"class": re.compile(r'mlcc_\d+_list')})
first_classs = []
for first_tag_name, first_class_content in zip(first_tag_names,
first_class_contents):
first_class_name = re.match(r'(.*?) （.*?',
first_tag_name.text).group(1)
# first_class_name = first_tag_name.text.replace(' ', '')
first_class = (first_class_name, first_class_content)
first_classs.append(first_class)
return first_classs
def __init__(self, product_url):
self.url = product_url
html_analyse = HtmlAnalyse(self.url)
self.bs_content = html_analyse.get_bs_contents()
def get_page_url(self, second_category):
first_category_name, second_category_name, second_category_url = second_category
while True:
try:
html_analyse = HtmlAnalyse(second_category_url,
proxy=self.proxy_ip)
bs_content = html_analyse.get_bs_contents()
break
except Exception as e:
print(sys._getframe().f_code.co_name, e)
self.proxy_ip = self.proxy_pool.get()
ul_tag = bs_content.find(name="ul", attrs={"class": "brcategories"})
third_category_tags = ul_tag.find_all(
name="div", attrs={"class": "rsGARealEstate"})
for third_category_tag in third_category_tags:
third_category_name = third_category_tag.a.text
third_category_url = Rs_Pre_Url + third_category_tag.a.get("href")
while True:
try:
html_analyse = HtmlAnalyse(third_category_url,
proxy=self.proxy_ip)
bs_content = html_analyse.get_bs_contents()
break
except Exception as e:
print(sys._getframe().f_code.co_name, e)
self.proxy_ip = self.proxy_pool.get()
try:
page_tag = bs_content.find(name="div",
attrs={
"class": "viewProdDiv"
}).text
except Exception as e:
print(third_category_url, e, "找不到page_tag")
continue
flag = re.match(r".*?共(.*?)个", page_tag)
page_count = int(int(flag.group(1).strip()) / 20 + 1)
for page_num in range(int(page_count)):
page_url = third_category_url + "?pn=" + str(page_num + 1)
while True:
try:
html_analyse = HtmlAnalyse(page_url,
proxy=self.proxy_ip)
bs_content = html_analyse.get_bs_contents()
break
except Exception as e:
print(sys._getframe().f_code.co_name, e)
self.proxy_ip = self.proxy_pool.get()
component_url_tags = bs_content.find_all(
name="a", attrs={"class": "tnProdDesc"})
page_attributes = []
for component_url_tag in component_url_tags:
component_url = Rs_Pre_Url + component_url_tag.get("href")
union_category_name = second_category_name + "---" + third_category_name
page_attribute = (first_category_name, union_category_name,
component_url)
page_attributes.append(page_attribute)
#
threadingpool = ThreadingPool(4)
threadingpool.multi_process(self.thread_go, page_attributes)
# for page_attribute in page_attributes:
# self.thread_go(page_attribute)
continue
def thread_go(self, page_attributes):
cc_unit, cc_kiname, cc_url = page_attributes
html_analyse = HtmlAnalyse(cc_url)
while True:
try:
bs_content = html_analyse.get_bs_contents()
break
except Exception as e:
print(sys._getframe().f_code.co_name, e)
brand_tag = bs_content.find(name="span", attrs={"itemprop": "brand"})
name_tag = bs_content.find(name="span", attrs={"itemprop": "mpn"})
if not brand_tag or not name_tag:
return
cc_brandname = brand_tag.text.strip()
cc_code = name_tag.text.strip()
img_tag = bs_content.find(name="img", attrs={"itemprop": "image"})
if not img_tag:
cc_img = ""
else:
cc_img = Rs_Pre_Url + img_tag.get("src")
attach_tag = bs_content.find(
name="a",
attrs={"onclick": re.compile(r"window\.open\('http://docs")})
if not attach_tag:
cc_attach = ""
else:
attach_name = attach_tag.get("onclick")
try:
cc_attach = re.match(r"window\.open\('(.*?\.pdf)'\)",
attach_name).group(1)
except Exception as e:
print(sys._getframe().f_code.co_name, e)
cc_attach = ""
component = (cc_code, cc_brandname, cc_unit, cc_kiname, cc_url,
cc_attach, cc_img)
# 器件属性
while True:
try:
orcl_conn = OracleSave(1000005)
orcl_conn.component_insert(component)
component_properties = []
tr_tags = bs_content.find_all(
name="tr", attrs={"class": re.compile(r"dr-table-row")})
for tr_tag in tr_tags:
td_tags = tr_tag.find_all(name="td")
parameter_name = td_tags[1].text
parameter_value = td_tags[2].text
component_property = (parameter_name, parameter_value)
component_properties.append(component_property)
orcl_conn.properties_insert(component_property)
orcl_conn.commit()
break
except Exception as e:
print(sys._getframe().f_code.co_name, e)
finally:
orcl_conn.conn.close()
"""
@description:
@author: RoyalClown
@date: 2017/3/20
"""
from Lib.NetCrawl.HtmlAnalyse import HtmlAnalyse
html_analyse = HtmlAnalyse("http://china.rs-online.com/web/p/igbt-transistors/7965064/")
bs_content = html_analyse.get_bs_contents()
print(bs_content)
def __init__(self, img_url):
self.img, self.url = img_url
html_analyse = HtmlAnalyse(self.url)
self.bs_content = html_analyse.get_bs_contents()
def __init__(self, url="http://www.azurewave.com/product_a001_1.asp"):
self.url = url
html_analyse = HtmlAnalyse(self.url)
self.bs_content = html_analyse.get_bs_contents()
def __init__(self, url, code):
self.url = url
self.code = code
html_analyse = HtmlAnalyse(self.url)
self.bs_content = html_analyse.get_bs_contents()