def get_all_links_from(channel):
for num in range(1, 3):
if channel.split('/')[-2] == 'shoujihaoma': #不爬取手机号码商品信息
pass
else:
page_parsing.get_links_from(channel, num, 'a1') #爬取个人卖家
page_parsing.get_links_from(channel, num, 'a2') #爬取商城卖家
def get_all_links_from(channel):
for num in range(1,3):
if channel.split('/')[-2] == 'shoujihaoma':#不爬取手机号码商品信息
pass
else:
page_parsing.get_links_from(channel,num,'a1')#爬取个人卖家
page_parsing.get_links_from(channel,num,'a2')#爬取商城卖家
def get_all_links_from(channel):
# get page 1 item information
for num in range(1, 5):
page_parsing.get_links_from(channel, num)
# remove the overlap links
sort_links_list = list(set(page_parsing.links_list))
all_links.append(sort_links_list)
def get_links_from_urllist(channel):
for num in range(1, 101):
try:
while True:
if get_links_from(channel, num) == None:
break
else:
get_links_from(channel, num)
except KeyboardInterrupt:
break
def get_all_links_from(channel):
for num in range(1, 100): #100为上限，如果设置上限过大会导致不断重复刷新
get_links_from(channel, num)
def get_alt_links_from(channel):
for run in range(1, 5):
get_links_from(channel, run)
def get_all_links_from(channel):
for i in range(1,101):
info = get_links_from(channel,i)
if info == 'none':
break
def get_all_links_from(channel):
for num in range(1,100):#100为上限，如果设置上限过大会导致不断重复刷新
get_links_from(channel,num)
def get_all_links_from(start_url):
for page_num in range(1, 101):
get_links_from(start_url, page_num)
def get_all_links_from(channel):
#获取每个频道页的商品链接
for num in range(1, 101):
get_links_from(channel, num)
def get_all_links_from(channel):
for who_sell in range(1, 3):
for page in range(1, 201):
page_parsing.get_links_from(channel, page, who_sell)
def get_all_links_from(channel):
for i in range(1, 100): #把所有网页链接爬取下来
get_links_from(channel, i, 'a') #a代表商家,o代表用户
def get_all_links_from(channel):
for num in range(1,101):
get_links_from(channel,num)
def get_all_links_from(channel):
for i in range(1, 101):
info = get_links_from(channel, i)
if info == 'none':
break
def get_all_links_from(category):
for num in range(1, 101):
get_links_from(category, num)
def get_all_links_from(channel):
for who_sell in range(1, 3):
for page in range(1, 201):
page_parsing.get_links_from(channel, page, who_sell)
def get_all_links_from(channel):
try:
for i in range(1, 100):
get_links_from(channel, i)
except TimeoutError:
pass
def get_all_link_from(channel):
for page in range(1, 101):
get_links_from(channel, page)
def get_all_links_from(channel):
for i in range(1, 100):
get_links_from(channel, i)
def get_all_links_from(channel):
for num in range(1, 20):
time.sleep(3)
get_links_from(channel, num)
def get_all_links_from(chanel):
for page_num in range(1, 201):
get_links_from(chanel, page_num)
from multiprocessing import Pool
from page_parsing import get_links_from
if __name__ == '__main__':
pool = Pool()
for num in range(1, 90):
pool.apply_async(get_links_from(num))
def get_all_link_from(channel):
for num in range(1, 101):
if get_links_from(channel, num) == "meiyou":
if (num == 1):
none_url_list.insert_one({"channel": channel})
break
def get_all_links_from(channel):
for i in range(1, 100):
get_links_from(channel, i)
def get_all_links_from(channel):
for num in range(2, 67):
get_links_from(channel, num)
def get_all_links_from(channel):
for num in range(1, 101):
get_links_from(channel, num)
def get_all_links_from(channel): # 抓取所有类型商品及页面详情
for num in range(1, 101):
get_links_from(channel, num)