def update_file_path(self,item,root_dir,sub_dir): '''改变网页的名字和路径''' item.idx=funcs.get_md52int(item.url) #计算文件的id号 # domain_name=funcs.url2domain(item.url) # ext=funcs.get_url_ext(item.url) #获取文件扩展名 # if ext in ('css','js'): # file_path=root_dir+self.url_prefix_delta+'/css_js'+sub_dir #css和js不用改名,放在css_js文件夹 # elif ext in settings.S_img_ext: # file_path=root_dir+self.url_prefix_delta+'/'+domain_name+sub_dir # else: #将html文件名改成id号 # sub_dir=funcs.url2path(item.url) # dir_path=os.path.dirname(root_dir+self.url_prefix_delta+sub_dir) # file_path=dir_path+'/'+str(item.idx)+'.html' turl, anchor = funcs.get_url_anchor( item.url ) #传True,表示转换的是文件保存的路径 sub_dir = funcs.transform_url( turl, True ) #file_path=os.path.dirname(root_dir+self.url_prefix_delta+sub_dir) file_path=root_dir + sub_dir #print file_path return item,file_path
def modify_tree(self,item): '''更改网页链接''' item.soup=self.both_need_clear(item.soup) #删除csdn不需要的东西 if settings.S_target_website=='csdn': item.soup=self.csdn_clear(item.soup) #删除iteye不需要的东西 if settings.S_target_website=='iteye': item.soup=self.iteye_clear(item.soup) #修改a链接 a_links=item.soup.find_all('a',href=True) for a in a_links: href=a.get('href','') full_href = urlparse.urljoin(item.url,href) a['href'] = funcs.transform_url( full_href ) # if '#' in href: # url_parts=href.split('#') #去掉锚点 # turl=url_parts[0].strip() # if len(url_parts)>=2: # anchor=url_parts[1].strip() # else: # anchor='' # else: # turl=href # anchor='' # if turl=='' or turl.startswith('javascript') or turl.startswith('#'): # continue # if funcs.valid_url(item.url,turl): # c_href=urlparse.urljoin(item.url,turl) # if len(re.findall('/',c_href))==2: #使http://www.csdn.net变成http://www.csdn.net/ # c_href=c_href+'/' # # sub_dir=funcs.url2path(c_href) # c_idx=funcs.get_md52int(c_href) # c_href=self.url_prefix_main+sub_dir # dir_path=os.path.dirname(c_href) # c_href=dir_path+'/'+str(c_idx)+'.html' # if anchor: # a['href']=c_href+'#'+anchor # else: # a['href']=c_href #修改css链接 css_links=item.soup.find_all('link',href=True) for css in css_links: href=css.get('href','') full_href = urlparse.urljoin(item.url,href) if href.startswith('http://bits.wikimedia.org/'): css['href'] = funcs.transform_url( full_href ) else: css['href'] = '' # if href.startswith('//bits.wikimedia.org/'): # #wiki特殊的css链接,为php,因此需要按照a链接那样分析 # href = 'http:' + href # # if '#' in href: # url_parts=href.split('#') #去掉锚点 # turl=url_parts[0].strip() # if len(url_parts)>=2: # anchor=url_parts[1].strip() # else: # anchor='' # else: # turl=href # anchor='' # if turl=='' or turl.startswith('javascript') or turl.startswith('#'): # continue # if funcs.valid_url(item.url,turl): # c_href=urlparse.urljoin(item.url,turl) # if len(re.findall('/',c_href))==2: #使http://www.csdn.net变成http://www.csdn.net/ # c_href=c_href+'/' # # sub_dir=funcs.url2path(c_href) # c_idx=funcs.get_md52int(c_href) # c_href=self.url_prefix_main+sub_dir # dir_path=os.path.dirname(c_href) # c_href=dir_path+'/'+str(c_idx)+'.html' # if anchor: # css['href']=c_href+'#'+anchor # else: # css['href']=c_href # else: # if funcs.valid_url(item.url,href): # c_href=urlparse.urljoin(item.url,href) # sub_url=urlparse.urlparse(c_href)[2] #使http://www.csdn.net/../sdf变成http://www.csdn.net/sdf # if sub_url.startswith('/../'): # sub_url=sub_url[3:] # c_href=self.url_prefix_main+'/css_js'+sub_url # css['href']=c_href # #把无关的css链接去掉 # css['href'] = '' #修改js链接 js_links=item.soup.find_all('script',src=True) for js in js_links: href=js.get('script','') if funcs.valid_url(item.url,href): c_href=urlparse.urljoin(item.url,href) sub_url=urlparse.urlparse(c_href)[2] #使http://www.csdn.net/../sdf变成http://www.csdn.net/sdf if sub_url.startswith('/../'): sub_url=sub_url[3:] c_href=self.url_prefix_main+'/css_js'+sub_url js['script']=c_href return item