def process(url, batch_id, parameter, *args, **kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
#{'Host': 'zhidao.baidu.com'}
headers = {"Host": domain_name}
setattr(process, '_downloader', DownloadWrapper(CACHE_SERVER, headers))
if not hasattr(process, '_cache'):
setattr(process, '_cache',
Cache(batch_id.split('-', 1)[0] + '-json', CACHE_SERVER))
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout = int(timeout)
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout)
if kwargs and kwargs.get("debug"):
print(len(content), "\n", content[:1000])
if content is False:
return False
return True
def process(url, batch_id, parameter, manager, other_batch_process_time, *args,
**kwargs):
print(url)
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader',
DownloadWrapper(None, headers, REGION_NAME))
if not hasattr(process, '_regs'):
setattr(
process, '_regs', {
'homepage':
re.compile('http://www.sfda.gov.cn/WS01/(.*?)/$'),
'detail':
re.compile('http://www.sfda.gov.cn/WS01/(.*?)/(.*?).html')
})
if not hasattr(process, '_cache'):
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout = int(timeout)
gap = max(gap - other_batch_process_time, 0)
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
print label
if label == 'homepage':
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout,
refresh=True)
dom = lxml.html.fromstring(content)
total_content = dom.xpath('//td[@class="pageTdSTR15"]//text()')[0]
total_page = int(re.findall(u'共(\d+)页', total_content)[0])
for page in range(2, total_page):
print(page)
hrefs = dom.xpath('//td[@class="ListColumnClass15"]/a/@href')
urls = []
for href in hrefs:
href = re.sub(u'\.\.', u'', href) # 网址是以..开头的相对路径
href = 'http://www.sfda.gov.cn/WS01' + href
urls.append(href)
manager.put_urls_enqueue(batch_id, urls)
page_url = '{}index_{}.html'.format(url, page)
content = process._downloader.downloader_wrapper(
page_url, batch_id, gap, timeout=timeout, refresh=True)
dom = lxml.html.fromstring(content)
return True
elif label == 'detail':
return parse_page(url)
def process(url, batch_id, parameter, manager, other_batch_process_time, *args, **kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_cache'):
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
if not hasattr(process, '_regs'):
setattr(process, '_regs', {
'home_page' : re.compile('http://www.zyctd.com/zixun/'),
'list_page' : re.compile('http://www.zyctd.com/zixun-(\d+).html')
})
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout= int(timeout)
gap = max(gap - other_batch_process_time, 0)
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
print label
if label == 'home_page':
url_pattern = 'http://www.zyctd.com/zixun-{}.html'
content = process._downloader.downloader_wrapper(
url,
batch_id,
gap,
timeout = timeout,
encoding = 'utf-8',
refresh = True)
page_content = re.findall('var pageCount = parseInt\(\'(\d+)\'\)', content) # 在网页元素中没有，通过js代码段获得
if not page_content:
return False
page_num = int(page_content[0])
urls = []
for page in range(2,page_num): # 根据页码将所有页加入队列
urls.append(url_pattern.format(page))
manager.put_urls_enqueue(batch_id, urls)
result_list = parse_list_page(content) # 首页本身作为第一页也有新闻信息，也要进行分析
return process._cache.post(url, json.dumps(result_list, ensure_ascii=False), refresh=True)
elif label == 'list_page':
content = process._downloader.downloader_wrapper(
url,
batch_id,
gap,
timeout = timeout,
encoding = 'utf-8',
refresh = True)
result_list = parse_list_page(content)
return process._cache.post(url, json.dumps(result_list, ensure_ascii=False), refresh=True)
def process(url, batch_id, parameter, manager, *args, **kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader',
DownloadWrapper(None, headers, REGION_NAME))
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CacheS3(head + '-json-' + tail))
if not hasattr(process, '_regs'):
setattr(process, '_regs',
re.compile(urlparse.urljoin(SITE, 'search\?word=(.+)')))
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout = int(timeout)
today_str = datetime.now().strftime('%Y%m%d')
word = urllib.unquote(process._regs.match(url).group(1))
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str,
'/opt/service/log/').info('start download')
refresh = False
for _ in range(5):
try:
content = process._downloader.downloader_wrapper(
url,
batch_id,
gap,
timeout=timeout,
encoding='gb18030',
refresh=refresh)
if content == '':
return False
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str,
'/opt/service/log/').info('start parsing url')
result = parse_search_json_v0707(content, word)
break
except:
refresh = True
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str,
'/opt/service/log/').info('start post json')
return process._cache.post(url, json.dumps(result))
def process(url, batch_id, parameter, manager, *args, **kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader',
DownloadWrapper('s3', headers, REGION_NAME))
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout = int(timeout)
today_str = datetime.now().strftime('%Y%m%d')
get_logger(batch_id, today_str, '/opt/service/log/').info('start download')
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout,
encoding='gb18030')
if kwargs and kwargs.get("debug"):
print(len(content), "\n", content[:1000])
if content is False:
return False
content_urls = []
get_logger(batch_id, today_str, '/opt/service/log/').info('start parsing')
tree = lxml.html.fromstring(content)
urls = tree.xpath('//td[@class="title"]/a/@href')
if urls == []:
get_logger(batch_id, today_str,
'/opt/service/log/').info('start download2')
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout,
encoding='gb18030',
refresh=True)
if content is False:
return False
tree = lxml.html.fromstring(content)
urls = tree.xpath('//td[@class="title"]/a/@href')
for url in urls:
content_urls.append(urlparse.urljoin('http://data.eastmoney.com/',
url))
get_logger(batch_id, today_str,
'/opt/service/log/').info('start put content')
manager.put_urls_enqueue('dongcaigonggao-content-20160620', content_urls)
return True
def process(url, batch_id, parameter, manager, *args, **kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {"Host": domain_name}
setattr(process, '_downloader', DownloadWrapper('s3', headers, REGION_NAME))
if not hasattr(process, '_cache'):
setattr(process, '_cache', CacheS3(batch_id.split('-', 1)[0]+'-json'))
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout= int(timeout)
today_str = datetime.now().strftime('%Y%m%d')
get_logger(batch_id, today_str, '/opt/service/log/').info('start download content')
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout,
encoding='gb18030')
if kwargs and kwargs.get("debug"):
print(len(content), "\n", content[:1000])
if content is False:
return False
get_logger(batch_id, today_str, '/opt/service/log/').info('start parsing content')
begin = content.find('<div class="mainbox">')
end = content.find('<div id="footer">', begin)
tree = lxml.html.fromstring(content[begin:end])
title = tree.xpath('//div[@class="content"]/h4/text()')
if isinstance(title, list) and len(title) > 0:
title = title[0]
else:
title = None
public_date = tree.xpath('//div[@class="content"]/h5/text()')
if isinstance(public_date, list) and len(public_date) > 0:
public_date = public_date[0]
else:
public_date = None
body = tree.xpath('//div[@class="content"]//pre/text()')
if isinstance(body, list) and len(body) > 0:
body = body[0]
else:
body = None
notice_content = json.dumps({'url': url, 'title': title, 'public_date': public_date, 'body': body})
get_logger(batch_id, today_str, '/opt/service/log/').info('start post json')
ret = process._cache.post(url, notice_content)
return ret
def process(url, batch_id, parameter, manager, other_batch_process_time, *args,
**kwargs):
# 药材的详情页涉及2个部分：价格历史history和边栏sidebar，以下的ytw/second/是价格历史的url，返回一个大的json；
# 所以在最后处理的时候还要额外向另一个url发送一次请求，以获得边栏信息,由于要储存到同一个result.json中，因此不再放入队列，而是直接在process里完成
today_str = datetime.now().strftime('%Y%m%d')
get_logger(batch_id, today_str,
'/opt/service/log/').info('process {}'.format(url))
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_regs'):
setattr(
process,
'_regs',
{
'home':
re.compile('http://www.yt1998.com/variteyIndexInfo.html'),
'kind':
re.compile('http://www.yt1998.com/issueIndexInfo.html\?code='),
'history':
re.compile(
'http://www.yt1998.com/ytw/second/indexMgr/getIndexInfo.jsp\?code=(\d+)&type=1&varitey_name=(.*)'
) #这是价格历史的url
})
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
if not hasattr(process, '_next_patterns'):
setattr(
process,
'_next_patterns',
{
'home':
'http://www.yt1998.com/issueIndexInfo.html?code={}', #the format of kind
'kind':
'http://www.yt1998.com/ytw/second/indexMgr/getIndexInfo.jsp?code={}&type=1&varitey_name={}', #the format of history
'history':
'http://www.yt1998.com/variteyIndexInfo.html?varitey_code={}' #the format of sidebar
})
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout = int(timeout)
gap = max(gap - other_batch_process_time, 0)
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
get_logger(batch_id, today_str,
'/opt/service/log/').info('label : {}'.format(label))
if label in [
'home', 'kind'
]: #I handle home-page and kind-page in one code block cuz they are in same web format
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout,
encoding='utf-8',
refresh=True)
dom = lxml.html.fromstring(content)
dd_labels = dom.xpath('//dd')
urls = []
for single_dd in dd_labels:
rels = single_dd.xpath('.//@rel')
if not rels:
get_logger(batch_id, today_str, '/opt/service/log/').info(
'wrong rels content : {}'.format(rels))
continue
for rel in rels:
code = rel.split(
','
)[-2] #在home页面，rel的格式为 'Z,家种类' code为Z；在kind页面，rel格式为'Z,家种类,000001,枸杞'，code为000001
if label == 'home':
urls.append(process._next_patterns[label].format(code))
else: # label == 'kind'
name = str(rel.split(',')[-1])
urls.append(process._next_patterns[label].format(
code, urllib.quote(name)))
manager.put_urls_enqueue(batch_id, urls)
elif label == 'history': #开始提取单种药品数据
code = m.group(1)
name = urllib.unquote(m.group(2))
sidebar_url = process._next_patterns[label].format(code)
sidebar_content = process._downloader.downloader_wrapper(
sidebar_url,
batch_id,
gap,
timeout=timeout,
encoding='utf-8',
refresh=True)
sidebar_dom = lxml.html.fromstring(sidebar_content)
sidebar_label = sidebar_dom.xpath(
'//div[@class="box-con-r fr"]/table//tr')
if not isinstance(sidebar_label, list) or len(sidebar_label) != 19:
get_logger(batch_id, today_str,
'/opt/service/log/').info('not legal list!')
return False
sidebar_item = {} # 边栏信息
for index in range(1, 16):
line_content = sidebar_label[index].xpath(
'./td/text()') #line content格式为 权重比：0.0278、市净率：2.00...
parts = line_content[0].split(
'：') # chinese colon :left part as key,right part as value
sidebar_item[parts[0]] = parts[1]
line_content = sidebar_label[16].xpath(
'./th/text()') #最后更新时间的样式与其他不同，为th
parts = line_content[0].split('：')
sidebar_item[parts[0]] = parts[1]
history_content = process._downloader.downloader_wrapper(
url,
batch_id,
gap,
timeout=timeout,
encoding='utf-8',
refresh=True)
if history_content == '':
return False
get_logger(
batch_id, today_str,
'/opt/service/log/').info('history downloading finished')
history_item = json.loads(history_content)[
u'DayMonthData'] #从结果中提取每天数据
price_history = {} #价格历史
for raw_daily_data in history_item:
date = raw_daily_data[u'Date_time']
price = raw_daily_data[u'DayCapilization']
price_history[date] = price
result_item = {}
result_item['name'] = name
result_item['info'] = sidebar_item
result_item['price_history'] = price_history
result_item['source'] = sidebar_url
return process._cache.post(url,
json.dumps(result_item,
ensure_ascii=False),
refresh=True)
def process(url, batch_id, parameter, manager, other_batch_process_time, *args,
**kwargs):
print(url)
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader',
DownloadWrapper(None, headers, REGION_NAME))
if not hasattr(process, '_regs'):
setattr(
process, '_regs', {
'column_id':
re.compile('(\d+)'),
'pages_view':
re.compile(
'http://www.yt1998.com/ytw/second/marketMgr/query.jsp\?lmid=(\d+?)&(.*)'
)
})
if not hasattr(process, '_cache'):
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout = int(timeout)
gap = max(gap - other_batch_process_time, 0)
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
if label == 'column_id':
query_url = 'http://www.yt1998.com/ytw/second/marketMgr/query.jsp'
column_id = url
page_size = 10
data = {
'lmid': column_id, # 栏目id，lm是栏目的首字母! 9代表产地信息，1代表品种分析，3代表天天行情
# 'scid':'1', # 对于天天行情，存在scid=市场id，但是尝试不传递这个参数，就会返回所有市场的新闻。且在返回值内依然可以找到市场字段，不会丢失信息。
'pageIndex': '0',
'pageSize': page_size,
'times': '1', # 非必要参数
}
content = process._downloader.downloader_wrapper(query_url,
batch_id,
gap,
method='post',
data=data,
timeout=timeout,
encoding='utf-8',
refresh=True)
news_info = json.loads(content)
total = int(news_info[u'total']) # 得出新闻总数，以此生成子任务
url_pattern = 'http://www.yt1998.com/ytw/second/marketMgr/query.jsp?lmid={}×=1&pageIndex={}&pageSize={}'
urls = []
for index in range(0, total / page_size + 1):
url = url_pattern.format(column_id, index, page_size)
if not check_date_ok(url):
break
urls.append(url)
manager.put_urls_enqueue(batch_id, urls)
elif label == 'pages_view':
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
method='get',
timeout=timeout,
encoding='utf-8',
refresh=True)
item = json.loads(content)
news_data = item[u'data']
menu_dic = {
'1': u'品种分析',
'3': u'天天行情',
'9': u'产地信息',
}
result_list = []
detail_pattern = 'http://www.yt1998.com/hqMinute--{}.html'
for news in news_data:
result = {
'news_title': news[u'title'],
'news_url': detail_pattern.format(news[u'acid']),
'news_desc': news[u'cont'].strip(),
'news_date': news[u'dtm'],
'news_keyword_list':
[news[u'ycnam']], # ycname = 药材nam = 药材name 取名逻辑很复杂
'access_time': datetime.datetime.utcnow().isoformat(),
'market': news[u'market'],
}
result['news_type'] = menu_dic[news[u'lmid']]
if news[u'lmid'] == '3': # 天天行情为快讯，是短新闻，不用再去取正文
result['news_content'] = result['news_desc']
else: # 其它栏目进行正文爬取
result['news_content'] = get_news_content(
result['news_url'], batch_id, gap, timeout)
result_list.append(result)
return process._cache.post(url,
json.dumps(result_list,
ensure_ascii=False),
refresh=True)
def process(url, batch_id, parameter, manager, *args, **kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader',
DownloadWrapper(None, headers, REGION_NAME))
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CacheS3(head + '-json-' + tail))
if not hasattr(process, '_regs'):
setattr(
process, '_regs', {
'entity':
re.compile(
urlparse.urljoin(SITE,
'cndbpedia/api/entity\?mention=(.+)')),
'avp':
re.compile(
urlparse.urljoin(SITE,
'cndbpedia/api/entityAVP\?entity=(.+)')),
'info':
re.compile(
urlparse.urljoin(
SITE, 'cndbpedia/api/entityInformation\?entity=(.+)')),
'tags':
re.compile(
urlparse.urljoin(SITE,
'cndbpedia/api/entityTag\?entity=(.+)')),
})
content = process._downloader.downloader_wrapper(url,
batch_id,
0,
timeout=10,
encoding='utf-8')
if content == '':
return False
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
entity = urllib.unquote(m.group(1))
if label == 'entity':
urls = []
avpair_api = urlparse.urljoin(SITE,
'cndbpedia/api/entityAVP?entity={}')
info_api = urlparse.urljoin(
SITE, 'cndbpedia/api/entityInformation?entity={}')
tags_api = urlparse.urljoin(SITE,
'cndbpedia/api/entityTag?entity={}')
js = json.loads(content)
for ent in js[u'entity']:
if isinstance(ent, unicode):
ent = ent.encode('utf-8')
ent = urllib.quote(ent)
urls.append(avpair_api.format(ent))
urls.append(info_api.format(ent))
urls.append(tags_api.format(ent))
return urls
else:
data = json.dumps({entity: json.loads(content)})
return process._cache.post(url, data)
def process(url, batch_id, parameter, manager, other_batch_process_time, *args,
**kwargs):
home_page = 'http://app1.sfda.gov.cn/datasearch/face3/base.jsp?tableId=25&tableName=TABLE25&title=%B9%FA%B2%FA%D2%A9%C6%B7&bcId=124356560303886909015737447882'
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_reg'):
setattr(
process, '_reg', {
'detail':
re.compile(
'http://app1.sfda.gov.cn/datasearch/face3/content.jsp\?tableId=25&tableName=TABLE25&tableView=%B9%FA%B2%FA%D2%A9%C6%B7&Id=(\d+)'
),
})
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout = int(timeout)
gap = max(gap - other_batch_process_time, 0)
today_str = datetime.now().strftime('%Y%m%d')
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str,
'/opt/service/log/').info('start download')
data = {
'tableId': '25',
'State': '1',
'bcId': '124356560303886909015737447882',
'State': '1',
'curstart': 1, #here!
'State': '1',
'tableName': 'TABLE25',
'State': '1',
'viewtitleName': 'COLUMN167',
'State': '1',
'viewsubTitleName': 'COLUMN166,COLUMN170,COLUMN821',
'State': '1',
'tableView': '%E5%9B%BD%E4%BA%A7%E8%8D%AF%E5%93%81',
'State': '1',
}
if url == home_page:
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str,
'/opt/service/log/').info('start parsing url')
page = 1
while 1:
data['curstart'] = page
content = process._downloader.downloader_wrapper(
'http://app1.sfda.gov.cn/datasearch/face3/search.jsp',
batch_id,
gap,
method='post',
timeout=timeout,
refresh=True,
data=data)
# if page == 3:
# return
ids = re.findall(u'国产药品&Id=(\d+)', content)
if not ids:
break
url_pattern = 'http://app1.sfda.gov.cn/datasearch/face3/content.jsp?tableId=25&tableName=TABLE25&tableView=%B9%FA%B2%FA%D2%A9%C6%B7&Id={}'
urls = []
for drug_id in ids:
url = url_pattern.format(drug_id)
urls.append(url)
manager.put_urls_enqueue(batch_id, urls)
page += 1
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str, '/opt/service/log/').info(
'going to page{}'.format(page))
return
elif process._reg['detail'].match(url):
content = process._downloader.downloader_wrapper(
url,
batch_id,
gap,
timeout=timeout,
)
if content == '':
return False
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str,
'/opt/service/log/').info('start parsing url')
dom = lxml.html.fromstring(content)
table = dom.xpath('//tr')
item = {
'license_number':
table[1].xpath('./td')[1].xpath('./text()'), #[u'批准文号'],
'product_name_chs':
table[2].xpath('./td')[1].xpath('./text()'), #[u'产品名称'],
'product_name_eng':
table[3].xpath('./td')[1].xpath('./text()'), #[u'英文名称'],
'commodity_name_chs':
table[4].xpath('./td')[1].xpath('./text()'), #[u'商品名'],
'drug_form':
table[5].xpath('./td')[1].xpath('./text()'), #[u'剂型'],
'specification':
table[6].xpath('./td')[1].xpath('./text()'), #[u'规格'],
'manufacturer_chs':
table[7].xpath('./td')[1].xpath('./text()'), #[u'生产单位'],
'manuf_address_chs':
table[8].xpath('./td')[1].xpath('./text()'), #[u'生产地址'],
'category':
table[9].xpath('./td')[1].xpath('./text()'), #[u'产品类别'],
'license_data':
table[11].xpath('./td')[1].xpath('./text()'), #[u'批准日期'],
'standard_code':
table[12].xpath('./td')[1].xpath('./text()'), #[u'药品本位码'],
'standard_code_remark':
table[13].xpath('./td')[1].xpath('./text()'), #[u'药品本位码备注'],
'source': [url],
}
for k, v in item.iteritems():
if len(v) > 0:
item[k] = v[0]
else:
item[k] = None
return process._cache.post(url, json.dumps(item, ensure_ascii=False))
def process(url, batch_id, parameter, manager, other_batch_process_time, *args,
**kwargs):
# 药材的详情页涉及2个部分：价格历史history和边栏sidebar，以下的ytw/second/是价格历史的url，返回一个大的json；
# 所以在最后处理的时候还要额外向另一个url发送一次请求，以获得边栏信息,由于要储存到同一个result.json中，因此不再放入队列，而是直接在process里完成
today_str = datetime.now().strftime('%Y%m%d')
get_logger(batch_id, today_str,
'/opt/service/log/').info('process {}'.format(url))
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_regs'):
setattr(
process, '_regs', {
'list_view':
re.compile(
'http://www.yt1998.com/price/nowDayPriceQ\!getPriceList.do\?pageIndex=(\d+)&pageSize=(\d+)'
),
'detail_view':
re.compile(
'http://www.yt1998.com/ytw/second/priceInMarket/getPriceHistory.jsp\?ycnam=(.*)&guige=(.*)&chandi=(.*)&market=(.*)'
)
})
if not hasattr(process, '_sellerMarket_list'):
setattr(process, '_sellerMarket_list',
['', u'亳州市场', u'安国市场', u'玉林市场', u'成都市场'])
# http://www.yt1998.com/price/nowDayPriceQ!getPriceList.do?pageIndex=0&pageSize=500
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout = int(timeout)
gap = max(gap - other_batch_process_time, 0)
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
get_logger(batch_id, today_str,
'/opt/service/log/').info('label : {}'.format(label))
if label == 'list_view':
get_logger(batch_id, today_str, '/opt/service/log/').info(label)
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout,
encoding='utf-8',
refresh=True)
get_logger(batch_id, today_str,
'/opt/service/log/').info('download ok')
get_logger(batch_id, today_str,
'/opt/service/log/').info(len(content))
list_item = json.loads(content)
urls = []
for detail_item in list_item[u'data']:
detail_url_pattern = 'http://www.yt1998.com/ytw/second/priceInMarket/getPriceHistory.jsp?ycnam={}&guige={}&chandi={}&market={}'
ycnam = str(detail_item[u'ycnam'])
chandi = str(detail_item[u'chandi'])
market = str(detail_item[u'market'])
guige = str(detail_item[u'guige'])
detail_url = detail_url_pattern.format(urllib.quote(ycnam),
urllib.quote(guige),
urllib.quote(chandi),
urllib.quote(market))
urls.append(detail_url)
get_logger(batch_id, today_str,
'/opt/service/log/').info('len urls')
get_logger(batch_id, today_str,
'/opt/service/log/').info(len(urls))
manager.put_urls_enqueue(batch_id, urls)
total_num = int(list_item[u'total'])
pageIndex = int(m.group(1))
pageSize = int(m.group(2))
if pageIndex == 0:
print(total_num // pageSize)
for index in range(1, total_num // pageSize + 1):
get_logger(batch_id, today_str,
'/opt/service/log/').info('iiiiiindex')
get_logger(batch_id, today_str,
'/opt/service/log/').info(index)
list_pattern = 'http://www.yt1998.com/price/nowDayPriceQ!getPriceList.do?pageIndex={}&pageSize={}'
list_url = list_pattern.format(index, pageSize)
manager.put_urls_enqueue(batch_id, [list_url])
return True
elif label == 'detail_view':
get_logger(batch_id, today_str, '/opt/service/log/').info(label)
ycnam = urllib.unquote(m.group(1))
guige = urllib.unquote(m.group(2))
chandi = urllib.unquote(m.group(3))
market = urllib.unquote(m.group(4))
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout,
encoding='utf-8',
refresh=True)
get_logger(batch_id, today_str,
'/opt/service/log/').info(len(content))
history_item = json.loads(content)
get_logger(batch_id, today_str,
'/opt/service/log/').info('downloaded')
price_history = {}
for raw_daily_data in history_item[u'DayPriceData']:
date = raw_daily_data[u'Date_time']
price = raw_daily_data[u'DayCapilization']
price_history[date] = price
source_url = 'http://www.yt1998.com/priceHistory.html?keywords={}&guige={}&chandi={}&market={}'
get_logger(batch_id, today_str, '/opt/service/log/').info('source')
get_logger(batch_id, today_str, '/opt/service/log/').info(
len(process._sellerMarket_list))
result_item = {
'name': ycnam,
'productGrade': guige,
'productPlaceOfOrigin': chandi,
'sellerMarket': process._sellerMarket_list[int(market)],
'price_history': price_history,
'source': source_url.format(ycnam, guige, chandi, market),
}
print(result_item)
result_item['access_time'] = datetime.utcnow().isoformat(
) # 从上面source的赋值可看出每个item都对应不同的参数
return process._cache.post(url,
json.dumps(result_item,
ensure_ascii=False),
refresh=True)
def process(url, batch_id, parameter, manager, other_batch_process_time, *args, **kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
if not hasattr(process, '_regs'):
setattr(process, '_regs', {
'main': re.compile(r'http://china.chemnet.com/hot-product/(\w|\d+).html'),
'prd': re.compile(r'http://china.chemnet.com/product/pclist--(.+?)--0.html'),
'comps': re.compile(r'http://china.chemnet.com/product/search.cgi')
})
def safe_state(statement):
return statement[0] if statement else ''
def xpath_string(n):
return "//*[@id=\"main\"]/div[1]/div[1]/table/tr[" + str(n) + "]/td[2]/text()"
method, gap, js, timeout, data = parameter.split(':')
gap = float(max(0, float(gap) - other_batch_process_time))
timeout= int(timeout)
compspat = 'http://china.chemnet.com/product/search.cgi?skey={};use_cas=0;f=pclist;p={}'
today_str = datetime.now().strftime('%Y%m%d')
# if kwargs and kwargs.get("debug"):
# get_logger(batch_id, today_str, '/opt/service/log/').info('start download')
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout,
# encoding='gb18030',
refresh=True
)
# print(content)
if content == '':
get_logger(batch_id, today_str, '/opt/service/log/').info(url + ' no content')
return False
# if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str, '/opt/service/log/').info('start parsing url')
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
page = etree.HTML(content.replace('<sub>', '').replace('</sub>', ''))
if label == 'main':
# print("add chems")
chems = page.xpath("//*[@id=\"main\"]/div[1]/div[2]/dl/dd/ul/li/p[2]/a/@href") # links for chems in main page
chems = [ urlparse.urljoin(SITE, chem) for chem in chems]
get_logger(batch_id, today_str, '/opt/service/log/').info('adding chems urls into queue')
manager.put_urls_enqueue(batch_id, chems)
return True
elif label == 'prd':
chem_uri = m.group(1)
chem_name = page.xpath("//*[@id=\"main\"]/div[1]/div[1]/table/tr[1]/td[2]/text()")[0]
get_logger(batch_id, today_str, '/opt/service/log/').info(chem_name + " main page")
comps = page.xpath("//*[@id=\"main\"]/div[2]/div[2]/dl/dd/form/table/tr[1]/td[2]/a[1]")
pagetext = page.xpath("//*[@id=\"main\"]/div[2]/div[2]/dl/dd/h6/div/text()[1]")
# print(pagetext[0])
total = int(re.compile(r'共有(\d+)条记录').search(pagetext[0].encode('utf-8')).group(1))
total = total // 10 + 1 if total % 10 != 0 else total // 10
dic = {
u'source': url,
u'中文名称': page.xpath(xpath_string(1))[0] if page.xpath(xpath_string(1)) else '',
u'英文名称': page.xpath(xpath_string(2))[0] if page.xpath(xpath_string(2)) else '',
u'中文别名': page.xpath(xpath_string(3))[0] if page.xpath(xpath_string(3)) else '',
u'CAS_RN': page.xpath(xpath_string(4))[0] if page.xpath(xpath_string(4)) else '',
u'EINECS': page.xpath(xpath_string(5))[0] if page.xpath(xpath_string(5)) else '',
u'分子式': page.xpath(xpath_string(6))[0] if page.xpath(xpath_string(6)) else '',
u'分子量': page.xpath(xpath_string(7))[0] if page.xpath(xpath_string(7)) else '',
u'危险品标志': page.xpath(xpath_string(8))[0].strip() if page.xpath(xpath_string(8)) else '',
u'风险术语': page.xpath(xpath_string(9))[0].strip() if page.xpath(xpath_string(9)) else '',
u'安全术语': page.xpath(xpath_string(10))[0].strip() if page.xpath(xpath_string(10)) else '',
u'物化性质': page.xpath("//*[@id=\"main\"]/div[1]/div[1]/table/tr[11]/td[2]/p/text()") if page.xpath("//*[@id=\"main\"]/div[1]/div[1]/table/tr[11]/td[2]/p/text()") else [],
u'用途': page.xpath(xpath_string(12))[0] if page.xpath(xpath_string(12)) else '',
u'上游原料': page.xpath('//*[@id=\"main\"]/div[1]/div[1]/table/tr[14]/td[2]/a/text()') if page.xpath('//*[@id=\"main\"]/div[1]/div[1]/table/tr[14]/td[2]/a/text()') else [],
u'下游产品': page.xpath('//*[@id=\"main\"]/div[1]/div[1]/table/tr[15]/td[2]/a/text()') if page.xpath('//*[@id=\"main\"]/div[1]/div[1]/table/tr[15]/td[2]/a/text()') else [],
}
data = json.dumps(dic, encoding='utf-8', ensure_ascii=False)
new_urls = []
for t in range(total):
new_url = compspat.format(chem_uri, str(t))
get_logger(batch_id, today_str, '/opt/service/log/').info("new url" + new_url)
new_urls.append(new_url)
manager.put_urls_enqueue(batch_id, new_urls)
get_logger(batch_id, today_str, '/opt/service/log/').info('start posting prd page to cache')
return process._cache.post(url, data)
else:
chem_name = page.xpath("//*[@id=\"main\"]/div[1]/div[1]/table/tr[1]/td[2]/text()")[0]
total = len(page.xpath("//*[@id=\"main\"]/div[2]/div[2]/dl/dd/form")) # total num of suppliers
dic = ''
for i in range(1, total + 1):
c = safe_state(page.xpath("//*[@id=\"main\"]/div[2]/div[2]/dl/dd/form[{}]".format(str(i))))
if c is '':
break
comp = {}
comp[u'source'] = url
comp[u'chem_name'] = chem_name
comp[u'name'] = safe_state(c.xpath(".//table/tr[1]/td[2]/a[1]/text()"))
comp[u'tel'] = safe_state(c.xpath(".//table/tr[2]/td[2]/text()"))
comp[u'fax'] = safe_state(c.xpath(".//table/tr[3]/td[2]/text()"))
comp[u'website'] = safe_state(c.xpath(".//table/tr[4]/td[2]/a/text()"))
dic += json.dumps(comp, encoding='utf-8', ensure_ascii=False) + '\n'
dic = dic.strip()
get_logger(batch_id, today_str, '/opt/service/log/').info('start posting companies to cache')
return process._cache.post(url, dic)
def process(url, batch_id, parameter, manager, other_batch_process_time, *args,
**kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
if not hasattr(process, '_regs'):
setattr(
process, '_regs', {
'main':
re.compile(
r'http://www.zysj.com.cn/zhongyaocai/index__\d+.html'),
'prd':
re.compile(
r'http://www.zysj.com.cn/zhongyaocai/yaocai_\w/(.+?).html')
})
method, gap, js, timeout, data = parameter.split(':')
gap = float(max(0, float(gap) - other_batch_process_time))
timeout = int(timeout)
today_str = datetime.now().strftime('%Y%m%d')
# if kwargs and kwargs.get("debug"):
# get_logger(batch_id, today_str, '/opt/service/log/').info('start download')
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout)
# print(content)
if content == '':
# print("no content")
get_logger(batch_id, today_str,
'/opt/service/log/').info(url + ' no content')
return False
# content.encoding='gb18030'
# if kwargs and kwargs.get("debug"):
# get_logger(batch_id, today_str, '/opt/service/log/').info('start parsing url')
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
page = etree.HTML(content)
if label == 'main':
get_logger(batch_id, today_str,
'/opt/service/log/').info("adding Chinese Meds")
meds = page.xpath("//*[@id=\"list\"]/ul/li/a/@href"
) # links for meds in main page
meds = [urlparse.urljoin(SITE, med) for med in meds]
# print(meds[:5])
get_logger(batch_id, today_str,
'/opt/service/log/').info('adding Meds urls into queue')
manager.put_urls_enqueue(batch_id, meds)
return True
elif label == 'prd':
med_name = page.xpath("//*[@id=\"article\"]/h1/text()")[0]
get_logger(batch_id, today_str,
'/opt/service/log/').info(med_name + " main page")
# print(med_name,"main page")
book_list = []
dictionary = {}
books = content.split('<hr />') # 用来分开不同的药典
if len(books) == 2: # 只有一个药典的情况
books = [books[0]]
else: # 有多个药典的情况
books = books[1:-1]
for book in books:
page = etree.HTML(
book.replace('<strong>',
'').replace('</strong>', '').replace(
'<sub>', '').replace('</sub>', ''))
med_info = page.xpath("//p/text()")
data = {}
# data['source'] = url
dictionary['source'] = url
# data['access_time'] = datetime.utcnow().isoformat()
dictionary['access_time'] = datetime.utcnow().isoformat()
data_list = []
for info in med_info:
m = re.compile(r'【.+?】').match(info.encode('utf-8'))
if m:
prop = m.group(0)[3:-3]
cleaned = re.sub(r'【.+?】', '', info.encode('utf-8'))
data[prop] = cleaned
data_list.append({prop: cleaned})
else:
data[prop] += '\n' + info.encode('utf-8')
data_list[-1][prop] += '\n' + info.encode('utf-8')
book_name = data['摘录']
# dics[book_name] = data
book_list.append({book_name: data_list}) # 为了保持原书籍的顺序，使用列表结构
dictionary[data['药材名称']] = book_list
dictionary = json.dumps(dictionary,
encoding='utf-8',
ensure_ascii=False)
get_logger(
batch_id, today_str,
'/opt/service/log/').info('start posting prd page to cache')
return process._cache.post(url, dictionary)
def process(url, batch_id, parameter, manager, *args, **kwargs):
# 药材的详情页涉及2个部分：价格历史history和边栏sidebar，以下的ytw/second/是价格历史的url，返回一个大的json；
# 所以在最后处理的时候还要额外向另一个url发送一次请求，以获得边栏信息,由于要储存到同一个result.json中，因此不再放入队列，而是直接在process里完成
today_str = datetime.now().strftime('%Y%m%d')
get_logger(batch_id, today_str,
'/opt/service/log/').info('process {}'.format(url))
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader',
DownloadWrapper(None, headers, REGION_NAME))
if not hasattr(process, '_regs'):
setattr(
process,
'_regs',
{
'home':
re.compile('http://www.yt1998.com/variteyIndexInfo.html'),
'kind':
re.compile('http://www.yt1998.com/issueIndexInfo.html\?code='),
'history':
re.compile(
'http://www.yt1998.com/ytw/second/indexMgr/getIndexInfo.jsp\?code=(\d+)&type=1&varitey_name=(.*)'
) #这是价格历史的url
})
# if not hasattr(process, '_cache'):
# head, tail = batch_id.split('-')
# setattr(process, '_cache', CacheS3(head + '-json-' + tail))
if not hasattr(process, '_next_patterns'):
setattr(
process,
'_next_patterns',
{
'home':
'http://www.yt1998.com/issueIndexInfo.html?code={}', #the format of kind
'kind':
'http://www.yt1998.com/ytw/second/indexMgr/getIndexInfo.jsp?code={}&type=1&varitey_name={}', #the format of history
'history':
'http://www.yt1998.com/variteyIndexInfo.html?varitey_code={}' #the format of sidebar
})
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout = int(timeout)
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
get_logger(batch_id, today_str,
'/opt/service/log/').info('label : {}'.format(label))
if label in [
'home', 'kind'
]: #I handle home-page and kind-page in one code block cuz they are in same web format
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout,
encoding='utf-8',
refresh=True)
dom = lxml.html.fromstring(content)
dd_labels = dom.xpath('//dd')
urls = []
for single_dd in dd_labels:
rels = single_dd.xpath('.//@rel')
if not rels:
get_logger(batch_id, today_str, '/opt/service/log/').info(
'wrong rels content : {}'.format(rels))
continue
for rel in rels:
code = rel.split(
','
)[-2] #在home页面，rel的格式为 'Z,家种类' code为Z；在kind页面，rel格式为'Z,家种类,000001,枸杞'，code为000001
if label == 'home':
urls.append(process._next_patterns[label].format(code))
else: # label == 'kind'
name = str(rel.split(',')[-1])
urls.append(process._next_patterns[label].format(
code, urllib.quote(name)))
manager.put_urls_enqueue(batch_id, urls)
elif label == 'history': #开始提取单种药品数据
#由于之前的设计，传进来的是历史价格的url，在更新的时候已经用不到，但是为了尽量一致，减少变动，采用传入
#历史记录url，再提取其中的参数组成边栏url，发送请求得到当日价格的逻辑
code = m.group(1)
name = urllib.unquote(m.group(2))
sidebar_url = process._next_patterns[label].format(code)
sidebar_content = process._downloader.downloader_wrapper(
sidebar_url,
batch_id,
gap,
timeout=timeout,
encoding='utf-8',
refresh=True)
sidebar_dom = lxml.html.fromstring(sidebar_content)
sidebar_label = sidebar_dom.xpath(
'//div[@class="box-con-r fr"]/table//tr')
if not isinstance(sidebar_label, list) or len(sidebar_label) != 19:
get_logger(batch_id, today_str,
'/opt/service/log/').info('not legal list!')
return False
for index in range(1, 16):
line_content = sidebar_label[index].xpath(
'./td/text()') #line content格式为 权重比：0.0278、市净率：2.00...
parts = line_content[0].split(
'：') # chinese colon :left part as key,right part as value
if parts[0] == u'当前价格':
# print ('相等')
today_price = parts[1]
break
result_item = {}
result_item['today_price'] = today_price
result_item['name'] = name
result_item['url'] = sidebar_url
return True # 之后更改为新的cache
def process(url, batch_id, parameter, manager, other_batch_process_time, *args,
**kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
cookie = kwargs.get('cookie', None)
# cookie = "gr_user_id=0fceb70d-e0ab-4c16-8f21-d49b5d242b0e; PHPSESSID=ltro2cjbvonlg6mu4hupe7dcv1; CNZZDATA1254842228=371101890-1469690209-null%7C1472547698"
if cookie:
headers.update({'Cookie': cookie})
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_cache'):
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
if not hasattr(process, '_regs'):
setattr(
process, '_regs', {
'search':
re.compile(
urlparse.urljoin(SITE,
'search\?key=(.+?)&index=(\d+)&p=(\d+)')),
'detail':
re.compile(
urlparse.urljoin(
SITE,
'company_getinfos\?unique=(.+?)&companyname=(.+?)&tab=base'
)),
'invest':
re.compile(
urlparse.urljoin(
SITE,
'company_getinfos\?unique=(.+?)&companyname=(.+?)(?:&p=(\d+))?&tab=touzi(?:&box=touzi)?'
)),
})
method, gap, js, timeout, data = parameter.split(':')
gap = float(max(0, float(gap) - other_batch_process_time))
timeout = int(timeout)
today_str = datetime.now().strftime('%Y%m%d')
# if kwargs and kwargs.get("debug"):
# get_logger(batch_id, today_str, '/opt/service/log/').info('start download')
def reformat(info): # 将info按企查查页面顺序插入队列
temp = info['info']
del info['info']
info['info'] = []
info['info'].append(("统一社会信用码", temp['unified_social_credit_code']))
info['info'].append(("注册号", temp['registration_id']))
info['info'].append(("组织机构代码", temp['organization_code']))
info['info'].append(("经营状态", temp['status']))
info['info'].append(("公司类型", temp['business_type']))
info['info'].append(("成立日期", temp['begin']))
info['info'].append(("法定代表", temp['legal_person']))
info['info'].append(("注册资本", temp['registered_capital']))
info['info'].append(("营业期限", temp['end']))
info['info'].append(("登记机关", temp['registration_authority']))
info['info'].append(("发照日期", temp['approval_date']))
info['info'].append(("企业地址", temp['address']))
info['info'].append(("经营范围", temp['business_scope']))
return info
def parse_company_investment(tree): # 解析对外投资页面，将子公司存入sub_companies字段下
invest_dict = {'sub_companies': []}
for sub_company in tree.cssselect('.list-group a.list-group-item'):
sub_name = sub_company.cssselect(
'span.clear .text-lg')[0].text_content().strip()
href = sub_company.get('href')
province, key_num = href.rsplit('_', 2)[-2:]
invest_dict['sub_companies'].append({
'name': sub_name,
'key_num': key_num,
'province': province,
'href': href,
})
return invest_dict
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
method,
timeout=timeout,
encoding='utf-8')
# print(url, file=log_file)
cookie = kwargs.get('cookie', None)
if not cookie:
get_logger(batch_id, today_str,
'/opt/service/log/').info("No cookie in worker")
return False
if content == '':
get_logger(batch_id, today_str, '/opt/service/log/').info("no content")
content = requests.get(url, cookies={1: cookie}).text
if content:
# print("got content", file=log_file)
get_logger(batch_id, today_str,
'/opt/service/log/').info('got content')
if not content and url.endswith("tab=touzi&box=touzi"):
get_logger(batch_id, today_str,
'/opt/service/log/').info("void invest page")
return True
invest_pat = "http://www.qichacha.com/company_getinfos?unique={key_num}&companyname={name}&p={p}&tab=touzi&box=touzi"
main_pat = "http://www.qichacha.com/company_getinfos?unique={key_num}&companyname={name}&tab=base"
search_pat = "http://www.qichacha.com/search?key={name}&index=0&p={p}"
parser = QiParser()
tree = lxml.html.fromstring(
content.replace('<em>', '').replace('</em>', ''))
# if kwargs and kwargs.get("debug"):
# print('start parsing url')
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
if label == 'search': # 搜索页面解析
comp_name = urllib.unquote(m.group(1))
dic = {'search_name': comp_name, 'names': []}
urls = []
if tree.cssselect('.table-search-list') and tree.cssselect(
'.tp2_tit a'):
items = tree.cssselect('.table-search-list')
for idx, i in enumerate(items):
if not i.xpath('.//*[@class=\"tp2_tit clear\"]/a/text()'):
continue
item = {}
item['name'] = i.xpath(
'.//*[@class=\"tp2_tit clear\"]/a/text()')[0]
# print(item['name'], file=log_file)
item['href'] = i.xpath(
'.//*[@class=\"tp2_tit clear\"]/a/@href')[0]
item['status'] = i.xpath(
'.//*[@class=\"tp5 text-center\"]/a/span/text()')[0]
item['key_num'] = item['href'].split('firm_')[1].split(
'.shtml')[0]
# print(item['key_num'], file=log_file)
if idx == 0 and comp_name == item[
'name']: # 若第一个搜索结果完全匹配则只添加第一个结果入待爬取队列
# get_logger(batch_id, today_str, '/opt/service/log/').info('appending', item['name'])
urls.append(
main_pat.format(key_num=item['key_num'],
name=item['name']))
urls.append(
invest_pat.format(key_num=item['key_num'],
name=item['name'],
p='1'))
break
elif idx < 3: # 如果第一个不完全匹配， 将前三个搜索结果加入待爬取队列
urls.append(
main_pat.format(key_num=item['key_num'],
name=item['name']))
urls.append(
invest_pat.format(key_num=item['key_num'],
name=item['name'],
p='1'))
dic['names'].append(item['name'])
if not urls:
return True
manager.put_urls_enqueue(batch_id, urls)
if not dic['names']:
return True
else: # 不完全匹配时将search_name与前三个搜索结果存入json用作别名映射
data = json.dumps(dic, encoding='utf-8', ensure_ascii=False)
return process._cache.post(url, data)
elif label == 'detail': # 解析详情页面
comp_name = urllib.unquote(m.group(2))
# print(comp_name, 'detail', file=log_file)
all_info = parser.parse_detail(tree)
all_info['name'] = comp_name
all_info['source'] = url
all_info['access_time'] = datetime.utcnow().isoformat()
all_info = parser.parser_patch(tree, all_info)
all_info = reformat(all_info)
data = json.dumps(all_info, encoding='utf-8', ensure_ascii=False)
get_logger(batch_id, today_str, '/opt/service/log/').info(data)
if not any([i[1] for i in all_info['info']]):
return False
return process._cache.post(url, data)
else: # 解析投资页面
comp_name = urllib.unquote(m.group(2))
key_num = m.group(1)
page = int(m.group(3))
pages = tree.xpath(".//a[@id=\"ajaxpage\"]/text()")
if '>' in pages:
urls = [
invest_pat.format(key_num=key_num,
name=comp_name,
p=str(page + 1))
]
manager.put_urls_enqueue(batch_id, urls)
invest_dict = parse_company_investment(tree)
# print(invest_dict, file=log_file)
if not invest_dict['sub_companies']:
return True
invest_dict['name'] = comp_name
invest_dict['source'] = url
invest_dict['access_time'] = datetime.utcnow().isoformat()
data = json.dumps(invest_dict,
encoding='utf-8',
ensure_ascii=False)
get_logger(batch_id, today_str, '/opt/service/log/').info(data)
return process._cache.post(url, data)
def process(url, batch_id, parameter, manager, other_batch_process_time, *args, **kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
if not hasattr(process, '_regs'):
setattr(process, '_regs', {
'main': re.compile(r'http://www.kmzyw.com.cn/bzjsp/biz_price_search/price_index_search.jsp'),
'prd': re.compile(r'http://www.kmzyw.com.cn/bzjsp/Biz_price_history/price_history_search.jsp\?name=(.*?)')
})
def timestamp2datetime(timestamp):
if isinstance(timestamp, (int, long, float)):
dt = datetime.utcfromtimestamp(timestamp)
else:
return "Not a valid timestamp"
mid = '-0' if dt.month < 10 else '-'
return str(dt.year) + mid + str(dt.month)
post_form = {
'pagecode': None,
# 'search_site': '%25E4%25BA%25B3%25E5%25B7%259E',
}
method, gap, js, timeout, data = parameter.split(':')
gap = float(max(0, float(gap) - other_batch_process_time))
timeout = int(timeout)
today_str = datetime.now().strftime('%Y%m%d')
get_logger(batch_id, today_str, '/opt/service/log/').info('start parsing url')
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
if label == 'main':
total_page = 10 # 初始化为一个较小的数，之后在获取页面内容后会更新此总页数
page_num = 1
while page_num < total_page + 1:
post_form['pagecode'] = page_num
# print(page_num)
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
method='post',
data=post_form,
timeout=timeout,
refresh=True
)
# print(content)
data = json.loads(content)
total_page = data['page'] # 从json中读出总页数
drugs = []
drug_url = 'http://www.kmzyw.com.cn/bzjsp/Biz_price_history/price_history_search.jsp?name={}'
for row in data['rows']:
# print(row['drug_name'])
drugs.append(drug_url.format(urllib.quote(str(row['drug_name'])).replace('%', '%25')))
manager.put_urls_enqueue(batch_id, drugs)
page_num += 1
return True
elif label == 'prd':
content = process._downloader.downloader_wrapper(
url,
batch_id,
gap,
timeout=timeout,
refresh=True
)
page = etree.HTML(content)
prd = page.xpath("/html/body/section[2]/h1/text()")[0]
idx = prd.index(u'品种')
prd = prd[:idx]
get_logger(batch_id, today_str, '/opt/service/log/').info(prd + " main page")
price_hist = page.xpath("/html/head/script[12]/text()")[0]
# print(price_hist)
data_pat = re.compile(r'series : \[(.*),marker')
m = data_pat.findall(price_hist)
dics = ''
if m:
# print(m[0])
data = m[0].split(',marker : { enabled : false ,radius : 3 } ,tooltip : { valueDecimals : 2 }},')
for d in data:
name = 'name'
data = 'data'
dic = eval(d + '}')
# print(dic)
cleaned = {}
cleaned['source'] = url
cleaned['specs'] = dic['name']
cleaned['name'] = prd
cleaned['data'] = [ (timestamp2datetime(int(price[0]) // 1000), price[1]) for price in dic['data'] ]
cleaned['access_time'] = datetime.utcnow().isoformat()
dics += json.dumps(cleaned, encoding='utf-8') + '\n'
else:
get_logger(batch_id, today_str, '/opt/service/log/').info('not match')
get_logger(batch_id, today_str, '/opt/service/log/').info('start posting prd page to cache')
return process._cache.post(url, dics)
def process(url, batch_id, parameter, manager, other_batch_process_time, *args,
**kwargs):
home_page = 'http://qy1.sfda.gov.cn/datasearch/face3/base.jsp?tableId=23&tableName=TABLE23&title=GMP%C8%CF%D6%A4&bcId=118715589530474392063703010776'
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_reg'):
setattr(
process, '_reg', {
'detail':
re.compile(
'http://qy1.sfda.gov.cn/datasearch/face3/content.jsp\?tableId=23&tableName=TABLE23&tableView=GMP%C8%CF%D6%A4&Id=(\d+)'
),
})
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout = int(timeout)
gap = max(gap - other_batch_process_time, 0)
today_str = datetime.now().strftime('%Y%m%d')
data = {
'tableId': '23',
'State': '1',
'bcId': '118715589530474392063703010776',
'State': '1',
'curstart': '4',
'State': '1',
'tableName': 'TABLE23',
'State': '1',
'viewtitleName': 'COLUMN152',
'State': '1',
'viewsubTitleName': 'COLUMN151',
'State': '1',
'tableView': 'GMP%E8%AE%A4%E8%AF%81',
'State': '1',
}
if url == home_page:
page = 1
while 1:
data['curstart'] = page
content = process._downloader.downloader_wrapper(
'http://qy1.sfda.gov.cn/datasearch/face3/search.jsp',
batch_id,
gap,
method='post',
timeout=timeout,
refresh=True,
data=data)
ids = re.findall(u'GMP认证&Id=(\d+)', content)
if not ids:
break
url_pattern = 'http://qy1.sfda.gov.cn/datasearch/face3/content.jsp?tableId=23&tableName=TABLE23&tableView=GMP%C8%CF%D6%A4&Id={}'
urls = []
for drug_id in ids:
url = url_pattern.format(drug_id)
urls.append(url)
manager.put_urls_enqueue(batch_id, urls)
page += 1
return True
elif process._reg['detail'].match(url):
content = process._downloader.downloader_wrapper(
url,
batch_id,
gap,
timeout=timeout,
)
if content == '':
return False
dom = lxml.html.fromstring(content)
table = dom.xpath('//tr')
item = {'source': url, 'access_time': datetime.utcnow().isoformat()}
tr_labels = dom.xpath('//tr')
for tr_label in tr_labels[1:]:
key = ''.join(tr_label.xpath('.//td[1]//text()')).strip()
value = ''.join(tr_label.xpath('.//td[2]//text()')).strip()
if value and key != u'注':
item[key] = value
return process._cache.post(url, json.dumps(item, ensure_ascii=False))
def process(url, batch_id, parameter, manager, other_batch_process_time, *args,
**kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
if not hasattr(process, '_regs'):
setattr(
process, '_regs', {
'main':
re.compile(
r'http://jiage.cngold.org/jinshubi/list_3640_(\d+).html'),
'info':
re.compile(
r'http://jiage.cngold.org/c/(\d+-\d+-\d+)/c(\d+).html'),
'index':
re.compile(r'http://jiage.cngold.org/jinshubi/index.html')
})
method, gap, js, timeout, data = parameter.split(':')
gap = float(max(0, float(gap) - other_batch_process_time))
timeout = int(timeout)
today_str = datetime.now().strftime('%Y%m%d')
if url == 'http://jiage.cngold.org/jinshubi/list_3640_1.html':
url = 'http://jiage.cngold.org/jinshubi/index.html'
# if kwargs and kwargs.get("debug"):
# get_logger(batch_id, today_str, '/opt/service/log/').info('start download')
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout)
# print(content)
if content == '':
get_logger(batch_id, today_str,
'/opt/service/log/').info(url + ' no content')
return False
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
# print("not match")
continue
page = etree.HTML(content)
if label == 'index':
prices = page.xpath(".//ul[@class='list_baojia']/li/a/@href")
# get_logger(batch_id, today_str, '/opt/service/log/').info(str(prices))
manager.put_urls_enqueue(batch_id, prices[:3])
return True
elif label == 'info':
dic = {}
datestr = m.group(1)
table = page.xpath(".//table//td/text()")
table = [t.strip() for t in table]
dic[u'产品名称'] = table[0]
dic[u'产品价格'] = table[1]
dic[u'价格单位'] = table[2]
dic[u'涨跌'] = table[3]
dic[u'日期'] = datestr
dic[u'source'] = url
dic[u'access_time'] = datetime.utcnow().isoformat()
data = json.dumps(dic, ensure_ascii=False)
# get_logger(batch_id, today_str, '/opt/service/log/').info(data)
return process._cache.post(url, data)
def process(url, batch_id, parameter, manager, other_batch_process_time, *args,
**kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
if not hasattr(process, '_regs'):
setattr(
process, '_regs', {
'main':
re.compile(
r'http://chem.100ppi.com/price/plist-(\d+)(-{1,3})(\d+).html'
),
'prd':
re.compile(r'http://www.100ppi.com/price/detail-(\d+).html')
})
def safe_state(statement):
return statement[0] if statement else ''
method, gap, js, timeout, data = parameter.split(':')
gap = float(max(0, float(gap) - other_batch_process_time))
timeout = int(timeout)
today_str = datetime.now().strftime('%Y%m%d')
# print(url)
# if kwargs and kwargs.get("debug"):
# get_logger(batch_id, today_str, '/opt/service/log/').info('start download')
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout)
# print(content)
if content == '':
get_logger(batch_id, today_str,
'/opt/service/log/').info(url + ' no content')
return False
# content.encoding='gb18030'
# if kwargs and kwargs.get("debug"):
# get_logger(batch_id, today_str, '/opt/service/log/').info('start parsing url')
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
page = etree.HTML(content)
if label == 'main':
# print("adding chem prds")
prd_links = page.xpath('//table/tr/td[1]/div/a/@href')
if not prd_links:
# print('end of pages')
get_logger(batch_id, today_str,
'/opt/service/log/').info('end of pages')
return True
next_pat = re.compile(r'plist-(\d+)(-{1,3})(\d+).html')
current = next_pat.search(url)
current = str(int(current.group(3)) + 1)
next_page = url[:url.rfind('-') + 1] + current + '.html'
prd_links.append(urlparse.urljoin(SITE, next_page))
get_logger(batch_id, today_str,
'/opt/service/log/').info('||'.join(prd_links) +
' added to queue')
manager.put_urls_enqueue(batch_id, prd_links)
return True
else:
data = {}
data['name'] = page.xpath(
"/html/body/div[8]/div[1]/span[2]/text()")[0]
# print(data['name'], 'prd page')
data['source'] = url
# data['prd_header'] = page.xpath("//div[@class=\"mb20\"]/table/tr/th/text()")
# data['prd_infos'] = page.xpath("//div[@class=\"mb20\"]/table/tr/td/text()")
prd_header = page.xpath(
"/html/body/div[8]/div[2]/div[1]/div[1]/h3/text()")[0]
idx_left, idx_right = prd_header.find(u'('), prd_header.find(u')')
data[u'报价类型'] = prd_header[idx_left + 1:idx_right]
data[u'报价机构'] = page.xpath(
"/html/body/div[8]/div[2]/div[2]/div[2]/table/tr[1]/td/h3/text()"
)[0].strip()
data[u'商品报价'] = safe_state(
page.xpath("//div[@class=\"mb20\"]/table/tr[1]/td[1]/text()"))
data[u'发布时间'] = safe_state(
page.xpath("//div[@class=\"mb20\"]/table/tr[1]/td[2]/text()"))
data[u'出产地'] = safe_state(
page.xpath("//div[@class=\"mb20\"]/table/tr[2]/td[1]/text()"))
data[u'有效期'] = safe_state(
page.xpath("//div[@class=\"mb20\"]/table/tr[2]/td[2]/text()"))
data[u'仓储地'] = safe_state(
page.xpath("//div[@class=\"mb20\"]/table/tr[3]/td[1]/text()"))
data[u'包装说明'] = safe_state(
page.xpath("//div[@class=\"mb20\"]/table/tr[3]/td[2]/text()"))
data[u'生产厂家'] = safe_state(
page.xpath(
"/html/body/div[8]/div[2]/div[1]/div[2]/div/div[2]/text()")
)
info = {}
table_header = page.xpath(
"//table[@class=\"mb20 st2-table tac\"]/tr/th/text()")
table_content = page.xpath(
"//table[@class=\"mb20 st2-table tac\"]/tr/td/text()")
for header, cont in zip(table_header, table_content):
info[header] = cont
data[u'详细信息'] = info
contact = {}
contact[u'联系人'] = safe_state(
page.xpath(
"//div[@class=\"connect\"]/table/tr[2]/td[2]/text()"))
contact[u'电话'] = safe_state(
page.xpath(
"//div[@class=\"connect\"]/table/tr[3]/td[2]/text()"))
contact[u'传真'] = safe_state(
page.xpath(
"//div[@class=\"connect\"]/table/tr[4]/td[2]/text()"))
contact[u'邮件'] = safe_state(
page.xpath(
"//div[@class=\"connect\"]/table/tr[5]/td[2]/text()"))
contact[u'手机'] = safe_state(
page.xpath(
"//div[@class=\"connect\"]/table/tr[6]/td[2]/text()"))
contact[u'地址'] = safe_state(
page.xpath(
"//div[@class=\"connect\"]/table/tr[7]/td[2]/text()"))
contact[u'网址'] = safe_state(
page.xpath(
"//div[@class=\"connect\"]/table/tr[8]/td[2]/text()"))
data[u'联系方式'] = contact
# print(json.dumps(data, encoding='utf-8', ensure_ascii=False))
dics = json.dumps(data, encoding='utf-8', ensure_ascii=False)
get_logger(batch_id, today_str,
'/opt/service/log/').info(dics + ' saved to S3')
return process._cache.post(url, dics)
def process(url, batch_id, parameter, manager, other_batch_process_time, *args,
**kwargs):
today_str = datetime.now().strftime('%Y%m%d')
get_logger(batch_id, today_str,
'/opt/service/log/').info('process {}'.format(url))
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_regs'):
setattr(
process, '_regs', {
'list_view':
re.compile(
'http://www.yt1998.com/price/nowDayPriceQ\!getPriceList.do\?pageIndex=(\d+)&pageSize=(\d+)'
),
'detail_view':
re.compile(
'http://www.yt1998.com/ytw/second/priceInMarket/getPriceHistory.jsp\?ycnam=(.*)&guige=(.*)&chandi=(.*)&market=(.*)'
)
})
# http://www.yt1998.com/price/nowDayPriceQ!getPriceList.do?pageIndex=0&pageSize=20
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
method, gap, js, timeout, data = parameter.split(':')
gap = float(max(0, float(gap) - other_batch_process_time))
timeout = int(timeout)
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
get_logger(batch_id, today_str,
'/opt/service/log/').info('label : {}'.format(label))
if label == 'list_view':
get_logger(batch_id, today_str, '/opt/service/log/').info(label)
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout,
encoding='utf-8',
refresh=True)
get_logger(batch_id, today_str,
'/opt/service/log/').info('download ok')
list_item = json.loads(content)
for detail_item in list_item[u'data']:
detail_item[u'access_time'] = datetime.utcnow().isoformat()
total_num = int(list_item[u'total'])
pageIndex = int(m.group(1))
pageSize = int(m.group(2))
if pageIndex == 0:
for index in range(1, total_num // pageSize + 1):
get_logger(batch_id, today_str,
'/opt/service/log/').info('index:')
get_logger(batch_id, today_str,
'/opt/service/log/').info(index)
list_pattern = 'http://www.yt1998.com/price/nowDayPriceQ!getPriceList.do?pageIndex={}&pageSize={}'
list_url = list_pattern.format(index, pageSize)
manager.put_urls_enqueue(batch_id, [list_url])
return process._cache.post(url,
json.dumps(list_item,
ensure_ascii=False),
refresh=True)
def process(url, batch_id, parameter, manager, other_batch_process_time, *args,
**kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
if not hasattr(process, '_regs'):
setattr(
process, '_regs', {
'main':
re.compile(
r'http://www.sge.com.cn/xqzx/mrxq/index_(\d+).shtml'),
'info':
re.compile(r'http://www.sge.com.cn/xqzx/mrxq/(\d+).shtml')
})
method, gap, js, timeout, data = parameter.split(':')
gap = float(max(0, float(gap) - other_batch_process_time))
timeout = int(timeout)
today_str = datetime.now().strftime('%Y%m%d')
# print(url)
# if kwargs and kwargs.get("debug"):
# get_logger(batch_id, today_str, '/opt/service/log/').info('start download')
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout)
# print(content)
if content == '':
get_logger(batch_id, today_str,
'/opt/service/log/').info(url + ' no content')
return False
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
# print("not match")
continue
page = etree.HTML(content)
if label == 'main':
get_logger(batch_id, today_str,
'/opt/service/log/').info('in list page')
urls = page.xpath(".//ul[@id='zl_list']/li/a/@href")
urls = [urlparse.urljoin(SITE, list_url) for list_url in urls]
get_logger(batch_id, today_str,
'/opt/service/log/').info(str(urls))
# get_logger(batch_id, today_str, '/opt/service/log/').info('||'.join(prd_links) + ' added to queue')
manager.put_urls_enqueue(batch_id, urls)
return True
elif label == 'info':
dic = {}
date = page.xpath(".//h5[@class='con_h5']/text()")[0].split(
u'\xa0')[0]
header = page.xpath(
".//div[@id='page_con']/table/tbody/tr[1]/td//text()")
infos = page.xpath(
".//div[@id='page_con']/table/tbody/tr/td[1]//text()")
infos = [info.strip() for info in infos if info.strip()]
idx = -1
for index, prod in enumerate(list(infos)):
if u"Pt9995" in prod:
idx = str(index + 1)
break
if idx == -1:
return True
pt_infos = page.xpath(
".//div[@id='page_con']/table/tbody/tr[{}]/td//text()".format(
idx))
if not pt_infos:
get_logger(
batch_id, today_str,
'/opt/service/log/').info("No pt info on this page " + url)
return True
for col, value in zip(header, pt_infos):
dic[col] = value.strip()
dic[u'日期'] = date
dic[u'source'] = url
dic[u'access_time'] = datetime.utcnow().isoformat()
data = json.dumps(dic, ensure_ascii=False)
get_logger(batch_id, today_str, '/opt/service/log/').info(data)
return process._cache.post(url, data)
def process(url, batch_id, parameter, manager, *args, **kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader',
DownloadWrapper('s3', headers, REGION_NAME))
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CacheS3(head + '-json-' + tail))
if not hasattr(process, '_regs'):
setattr(
process, '_regs', {
'entity':
re.compile(
urlparse.urljoin(SITE,
'cndbpedia/api/entity\?mention=(.+)')),
'avp':
re.compile(
urlparse.urljoin(SITE,
'cndbpedia/api/entityAVP\?entity=(.+)')),
'info':
re.compile(
urlparse.urljoin(
SITE, 'cndbpedia/api/entityInformation\?entity=(.+)')),
'tags':
re.compile(
urlparse.urljoin(SITE,
'cndbpedia/api/entityTag\?entity=(.+)')),
})
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout = int(timeout)
today_str = datetime.now().strftime('%Y%m%d')
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str,
'/opt/service/log/').info('start download')
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout,
encoding='utf-8')
if content == '':
return False
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str,
'/opt/service/log/').info('start parsing url')
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
entity = urllib.unquote(m.group(1))
if label == 'entity':
urls = []
avpair_api = urlparse.urljoin(SITE,
'cndbpedia/api/entityAVP?entity={}')
info_api = urlparse.urljoin(
SITE, 'cndbpedia/api/entityInformation?entity={}')
tags_api = urlparse.urljoin(SITE,
'cndbpedia/api/entityTag?entity={}')
js = json.loads(content)
for ent in js[u'entity']:
if isinstance(ent, unicode):
ent = ent.encode('utf-8')
ent = urllib.quote(ent)
urls.append(avpair_api.format(ent))
urls.append(info_api.format(ent))
urls.append(tags_api.format(ent))
manager.put_urls_enqueue(batch_id, urls)
return True
else:
data = json.dumps({entity: json.loads(content)})
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str, '/opt/service/log/').info(
'start post {} json'.format(label))
return process._cache.post(url, data)
def process(url, batch_id, parameter, manager, other_batch_process_time, *args, **kwargs):
today_str = datetime.now().strftime('%Y%m%d')
get_logger(batch_id, today_str, '/opt/service/log/').info(url)
home_page = 'http://app1.sfda.gov.cn/datasearch/face3/base.jsp?tableId=36&tableName=TABLE36&title=%BD%F8%BF%DA%D2%A9%C6%B7&bcId=124356651564146415214424405468'
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process,'_reg'):
setattr(process, '_reg', {
'detail': re.compile('http://app1.sfda.gov.cn/datasearch/face3/content.jsp\?tableId=36&tableName=TABLE36&Id=(\d+)'),
})
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout= int(timeout)
gap = max(gap - other_batch_process_time, 0)
#if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str, '/opt/service/log/').info('start download')
data = {
'tableId' : '36',
'State' : '1',
'bcId' : '124356651564146415214424405468',
'State' : '1',
'State' : '1',
'tableName' : 'TABLE36',
'State' : '1',
'viewtitleName' : 'COLUMN361',
'State' : '1',
'viewsubTitleName' : 'COLUMN354,COLUMN355,COLUMN356,COLUMN823',
'curstart':'2',
'State' : '1',
'State' : '1',
}
if url == home_page:
#if kwargs and kwargs.get("debug"):
page = 1
while 1 :
time.sleep(gap)
get_logger(batch_id, today_str, '/opt/service/log/').info('start parsing url at page {}'.format(page))
data['curstart'] = page
content = process._downloader.downloader_wrapper('http://app1.sfda.gov.cn/datasearch/face3/search.jsp',
batch_id,
gap,
method = 'post',
timeout = timeout,
refresh = True,
data = data,
encoding = 'utf-8'
)
#get_logger(batch_id, today_str, '/opt/service/log/').info(content)
ids = re.findall(u'进口药品&Id=(\d+)', content)
get_logger(batch_id, today_str, '/opt/service/log/').info(ids)
if not ids:
get_logger(batch_id, today_str, '/opt/service/log/').info('End at {} pages'.format(page))
break
# if page == 3:
# break
get_logger(batch_id, today_str, '/opt/service/log/').info('ids : {}'.format(ids))
url_pattern = 'http://app1.sfda.gov.cn/datasearch/face3/content.jsp?tableId=36&tableName=TABLE36&Id={}'
urls = []
for drug_id in ids:
url = url_pattern.format(drug_id)
urls.append(url)
manager.put_urls_enqueue(batch_id, urls)
page += 1
get_logger(batch_id, today_str, '/opt/service/log/').info('going to page{}'.format(page))
return True
elif process._reg['detail'].match(url):
content = process._downloader.downloader_wrapper(
url,
batch_id,
gap,
timeout = timeout,
refresh = True
)
if content == '':
return False
dom = lxml.html.fromstring(content)
table = dom.xpath('//tr')
item = {
'license_number': table[1].xpath('./td')[1].xpath('./text()'), # [u'注册证号']
'old_license_number': table[2].xpath('./td')[1].xpath('./text()'), # [u'原注册证号']
'packaging_license_number': table[4].xpath('./td')[1].xpath('./text()'), # [u'分包装批准文号']
'company_chs': table[5].xpath('./td')[1].xpath('./text()'), # [u'公司名称（中文）']
'company_eng': table[6].xpath('./td')[1].xpath('./text()'), # [u'公司名称（英文）']
'product_name_chs': table[11].xpath('./td')[1].xpath('./text()'), # [u'产品名称（中文）']
'product_name_eng': table[12].xpath('./td')[1].xpath('./text()'), # [u'产品名称（英文）']
'commodity_name_chs': table[13].xpath('./td')[1].xpath('./text()'), # [u'商品名（中文）']
'commodity_name_eng': table[14].xpath('./td')[1].xpath('./text()'), # [u'商品名（英文）']
'drug_form': table[15].xpath('./td')[1].xpath('./text()'), # [u'剂型（中文）']
'specification': table[16].xpath('./td')[1].xpath('./text()'), # [u'规格（中文）']
'dosage': table[17].xpath('./td')[1].xpath('./text()'), # [u'包装规格（中文）']
'manufacturer_chs': table[18].xpath('./td')[1].xpath('./text()'), # [u'生产厂商（中文）']
'manufacturer_eng': table[19].xpath('./td')[1].xpath('./text()'), # [u'生产厂商（英文）']
'manuf_address_chs': table[20].xpath('./td')[1].xpath('./text()'), # [u'厂商地址（中文）']
'manuf_address_eng': table[21].xpath('./td')[1].xpath('./text()'), # [u'厂商地址（英文）']
'manuf_country_chs': table[22].xpath('./td')[1].xpath('./text()'), # [u'厂商国家/地区（中文）']
'manuf_country_eng': table[23].xpath('./td')[1].xpath('./text()'), # [u'厂商国家/地区（英文）']
'packaging_company_name': table[26].xpath('./td')[1].xpath('./text()'), # [u'分包装企业名称']
'packaging_company_address': table[27].xpath('./td')[1].xpath('./text()'), # [u'分包装企业地址']
'category': table[31].xpath('./td')[1].xpath('./text()'), # [u'产品类别']
'standard_code': table[32].xpath('./td')[1].xpath('./text()'), # [u'药品本位码']
'source' : [url], #设为list格式与之前字段统一，在下面的循环里一并取出
}
for k,v in item.iteritems():
if len(v) > 0:
item[k] = v[0]
else :
item[k] = None
return process._cache.post(url, json.dumps(item, ensure_ascii = False))