def generate_volume_links(self):
if self.JournalObj.volume_links_got:
pass
#假如数据库中已保存，直接读取即可，无需生成
for volume_area_link in ElsevierAllItemsPageParser(
html_source=request_with_random_ua(
self.url).text).volume_area_links:
#先分volume年份区间（十年）
#print('Elsevier Volume Area link:%s'%volume_area_link)
area_volume_links = ElsevierAllItemsPageParser(
html_source=request_with_random_ua(
volume_area_link).text).volume_links
area_volume_links.append(volume_area_link)
#得到该区间所有年份的page_url
self.volume_links.extend(area_volume_links)
def crawl_volume_page(self,
volume_item,
AllItemsPageParser,
JournalArticle,
use_tor=False,
check_pdf_url=True):
volume_link = self.handle_volume_link_for_multi_results(volume_item[0])
volume_db_id = volume_item[1]
if use_tor:
#此情况属于完全不可能直接获取pdf_url时使用，用远程服务器加速
html_source = request_with_proxy(volume_link).text
else:
html_source = request_with_random_ua(volume_link).text
parser = AllItemsPageParser(html_source)
try:
sections = parser.sections
except Exception as e:
print('[Error] JournalSpider:Page Invalid {}\n\
error_url {}'.format(str(e), volume_link))
return False
try:
volume_year = parser.volume_year
print('Volume_year:{}'.format(volume_year))
except AttributeError:
volume_year = None
print('\nPage Url: %s ' % volume_link)
if self.crawl_articles(sections, volume_year, volume_db_id,
JournalArticle, check_pdf_url) == False:
return False
if len(parser.sections) > 0 and self.debug == False:
self.mark_volume_ok(volume_db_id)
return True
return False
def generate_volume_links(self):
if self.JournalObj.volume_links_got:
return
#假如数据库中已保存，直接读取即可，无需生成
self.volume_links = IEEE_AllItemsPageParser(
html_source = request_with_random_ua(self.url).text
).volume_links
def page_amount(self):
if self.keyword:
self.result_amount = SearchPageParser(
html_source=request_with_random_ua(
self.url).text).result_amount
else:
self.result_amount = RankPageParser(self.url).result_amount
return int(self.result_amount / 50) + 1
def generate_volume_links(self):
if self.JournalObj.volume_links_got:
return
self.volumes_page_url = 'http://onlinelibrary.wiley.com' + BeautifulSoup(
request_with_random_ua(self.url).text, 'lxml').find(
'a', text='See all')['href']
#self.volumes_page_url = 'http://onlinelibrary.wiley.com/journal/10.1002/(ISSN)1096-987X/issues'
soup = BeautifulSoup(
request_with_random_ua(self.volumes_page_url).text, 'lxml')
years = [
int(a['href'].split('=')[-1])
for a in soup.find_all('a', id=re.compile('year_[0-9]+_link'))
]
pool = ThreadPool(16)
pool.map(self.get_volume_links_by_year, years)
pool.close()
pool.join()
def generate_abstract(self):
#访问abstract_url直接生成
try:
resp = request_with_random_ua(self.abstract_url)
self.abstract = BeautifulSoup(
resp.text, 'lxml').select_one('.paraText').text.strip()
except Exception as e:
pass
def generate_volume_links(self):
id = self.url.split('id=')[-1]
volumes_page_link = 'http://www.emeraldinsight.com/loi/{}'.format(id)
for i in range(10):
resp = request_with_random_ua(
url=volumes_page_link,timeout=10)
soup = BS(resp.text,'lxml')
self.volume_links = [ a['href'] for a in soup.select('.tocIssueLink') ]
if self.volume_links!=[]:
break
def get_volume_link_by_year(self,year):
volumes_page_url = '{}&year={}'.format(self.url,year)
print(volumes_page_url)
volume_page_links = [
'http://journals.lww.com{}'.format(a['href'])
for a in BS(request_with_random_ua(volumes_page_url).text,'lxml')\
.select_one('#ej-past-issues-detail-list')\
.find_all('a',text=re.compile('Volume'))
]
self.volume_links.extend(volume_page_links)
def generate_volume_links(self):
if self.JournalObj.volume_links_got:
return
#假如数据库中已保存，直接读取即可，无需生成
for page_num in range(1,SpringParser(
html_source=request_with_random_ua(self.url).text
).pages_amount+1):
page_url = 'http://link.springer.com/search/page/{}?facet-journal-id={}&showAll=true&facet-content-type=Article&sortOrder=newestFirst'\
.format(page_num,self.spring_journal_id)
self.volume_links.append(page_url)
def generate_volume_links(self):
#http://pubs.acs.org/journal/ancac3
if self.JournalObj.volume_links_got:
return
index = self.url.split('/')[-1]
volumes_page_url = 'http://pubs.acs.org/loi/{}'.format(index)
#print(volumes_page_url)
a_list = BeautifulSoup(
request_with_random_ua(volumes_page_url).text,
'lxml').select('.publicationTitle')
for i in range(20):
self.volume_links = [
a['href'] for a in BeautifulSoup(
request_with_random_ua(volumes_page_url).text,
'lxml').select('.publicationTitle')
]
print(self.volume_links)
if self.volume_links:
print('length:', len(self.volume_links))
break
def __init__(self, url, area_id=None, category_id=None, driver=None):
self.url = url
if category_id and area_id:
self.url += '?area={}&category={}&order=tr&type=j'.format(
area_id, category_id)
if driver:
driver.get(self.url)
time.sleep(2)
self.soup = BeautifulSoup(driver.page_source, 'lxml')
else:
resp = request_with_random_ua(self.url, timeout=20)
self.soup = BeautifulSoup(resp.text, 'lxml')
def generate_volume_links(self):
if self.JournalObj.volume_links_got:
return
name = self.url[:-1].split('/')[-1]
items_fist_page_url = 'http://{}.biomedcentral.com/articles?searchType=journalSearch&sort=PubDate&page=1'.format(
name)
print(items_fist_page_url)
pages_num = BioMedParser(html_source=request_with_random_ua(
url=items_fist_page_url, timeout=3).text).pages_amount
for i in range(1, pages_num + 1):
page_url = 'http://{}.biomedcentral.com/articles?searchType=journalSearch&sort=PubDate&page={}'\
.format(name,i)
self.volume_links.append(page_url)
def __init__(self, journal_sjr_id):
url = 'http://www.scimagojr.com/journalsearch.php?q={}&tip=sid&clean=0'.format(
journal_sjr_id)
print(url)
resp = request_with_random_ua(url)
self.journal_id = journal_sjr_id
self.soup = BeautifulSoup(resp.text, 'lxml')
self.trs = self.soup.find('tbody').find_all('tr')
self.info_dict = {}
for tr in self.trs:
tds = tr.find_all('td')
self.info_dict[tds[0].text] = tds[1]
#print(self.info_dict.keys())
self.cur = DB_CONNS_POOL.new_db_cursor()
def generate_volume_links(self):
if self.JournalObj.volume_links_got:
return
self.url = (self.url+'?mobile=0&desktopMode=true')\
.replace('default','issuelist')
soup = BS(request_with_random_ua(self.url).text,'lxml')
years = [
int(option.text) for option in
soup.select_one('#ej-article-action-toolbar-select').select('option')
]
pool = ThreadPool(8)
pool.map(self.get_volume_link_by_year,years)
pool.close()
pool.join()
def get_volume_links_by_year(self, year):
ajax_url = '{}/fragment?activeYear={}&SKIP_DECORATION=true'\
.format(self.volumes_page_url,year)
for i in range(5):
a_list = BeautifulSoup(
request_with_random_ua(ajax_url).text,
'lxml').select('.issue > a')
if a_list:
break
volume_links_by_year = [
'http://onlinelibrary.wiley.com' + a['href'] for a in a_list
]
if volume_links_by_year == []:
print('error ajax_url:{}'.format(ajax_url))
self.volume_links.extend(volume_links_by_year)
def crawl_per_page_result(self, url):
print(url)
if self.crawl_all:
if self.driver_pool_size > 0:
driverObj = self.drivers.get_one_free_driver()
drvier = driverObj.engine
else:
drvier = None
try:
parser = RankPageParser(url, drvier)
except Exception as e:
print(str(e))
try:
driverObj.status = 'free'
except:
pass
JournalItem = RankJournal
else:
html_source = request_with_random_ua(url).text
parser = SearchPageParser(html_source)
JournalItem = PublisherJournal
print(len(parser.sections))
for sec in parser.sections:
JournalItem(sec).save_to_db()
def handle_sciencedirect_url(self):
#print(self.url)
if 'sciencedirect' in self.url:
return
resp = request_with_random_ua(self.url)
soup = BeautifulSoup(resp.text, 'lxml')
print('---------------------')
if soup.find_all(text='404'):
raise Exception('404')
temp_url = soup.select_one('.cta-primary')['href']
temp_url2 = soup.find_all("a", href=re.compile("guide-for-authors"))
if temp_url2:
temp_url2 = temp_url2[0]['href']
print('temp_url:{}'.format(temp_url))
print('temp_url2:{}'.format(temp_url2))
for url in [temp_url2, temp_url]:
try:
journal_index = url.split('/')[0].split('-')
self.url = 'http://www.sciencedirect.com/science/journal/{}'\
.format(journal_index[0]+journal_index[1])
return
except Exception as e:
continue
print(str(e))
def generate_volume_links(self):
if self.JournalObj.volume_links_got:
return
self.volume_links = TaylorFrancisParser(
html_source=request_with_random_ua(self.url).text).volume_links