def period_set_cache(url_hash, batch_id, groups, content, refresh=False):
try:
assert len(batch_id) > 0
today = date.today()
yy = str(today.year)
absdir = os.path.join(HPCACHEDIR, yy, batch_id)
cache_file = os.path.join(absdir, url_hash)
path.makedir(absdir)
if refresh or not os.path.isfile(cache_file):
with open(cache_file, 'w') as fd:
fd.write(content)
now = datetime.now()
# use iso format rather than string format to make it more parsable
log_line = json.dumps({
'date': str(now),
'batch_id': batch_id,
'groups': groups,
# 'url': base64.urlsafe_b64decode(b64url),
})
cachelog.get_logger(batch_id, now.strftime('%Y%m%d'),
HPCACHEDIR).info(log_line)
except Exception as e:
return {'success': False, 'error': e}
return {'success': True}
def fs_set_cache(b64url, url_hash, batch_id, groups, content, refresh=False):
try:
assert len(batch_id) > 0
level1 = url_hash[0]
level2 = url_hash[-2:]
absdir = os.path.join(FSCACHEDIR, batch_id, 'raw', 'latest', level1,
level2)
cache_file = os.path.join(absdir, url_hash)
path.makedir(absdir)
if refresh or not os.path.isfile(cache_file):
with open(cache_file, 'w') as fd:
fd.write(content)
now = datetime.now()
# use iso format rather than string format to make it more parsable
log_line = json.dumps({
'date': str(now),
'batch_id': batch_id,
'groups': groups,
'url': base64.urlsafe_b64decode(b64url),
})
cachelog.get_logger(batch_id, now.strftime('%Y%m%d'),
FSCACHEDIR).info(log_line)
except Exception as e:
return {'success': False, 'error': e}
return {'success': True}
def run(self, *args, **kwargs):
""" end, background_cleansing, status:
None, None, begin
None, 1, begin cleaning
None, 0, finish cleaning
time, 0, begin delete
time, None, finish delete
None, 0, finish cleaning then exception
"""
checkout_cache = {}
for batch_id, queue_dict in self.manager.get_queue_with_priority():
queue = queue_dict['queue']
if Record.instance().is_finished(batch_id) is True:
# this queue and queue object in distributed_queues can be
# delete, but not needed. When run finish and
# background_cleansing finsh, this process end.
# BTW, worker instance will reboot every 10 minutes.
# If another worker delete queue, I don't need to do anything.
continue
background = queue.get_background_cleaning_status()
if background == '0':
pass
elif background is None or background == '1':
checkout_cache[batch_id] = queue_dict
today_str = datetime.now().strftime('%Y%m%d')
while len(checkout_cache) > 0:
removes = []
batch_urlid = {}
for batch_id, queue_dict in checkout_cache.iteritems(): # get url_ids from queue
get_logger(batch_id, today_str, '/opt/service/log/').info('begin get items from queue')
results = queue_dict['queue'].get(block=True, timeout=3, interval=1)
get_logger(batch_id, today_str, '/opt/service/log/').info('finish get items from queue')
if not results:
removes.append(batch_id)
continue
batch_urlid[batch_id] = results
[checkout_cache.pop(i) for i in removes]
while len(batch_urlid) > 0:
removes = []
for batch_id, results in batch_urlid.iteritems(): # download and process
if len(results) > 0:
url_id = results.pop()
other_batch_process_time = self.get_other_batch_process_time( set(batch_urlid.keys()) - set([batch_id]) )
start = time.time()
self.work(batch_id, checkout_cache[batch_id], url_id, other_batch_process_time, *args, **kwargs)
self.update_process_time_of_this_batch(batch_id, start)
else:
removes.append(batch_id)
[batch_urlid.pop(i) for i in removes]
def s3_put_cache(b64url, url_hash, batch_id, groups, content, refresh=False):
"""
:param groups: e.g. 'g1, g2'
"""
def put_cache(batch_key, filename, content):
try:
ret = S3.Object(batch_key, filename).put(Body=content)
if ret['ResponseMetadata']['HTTPStatusCode'] == 200:
return {'success': True}
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'NoSuchBucket':
ret = create_bucket(batch_key)
if type(ret) == str:
return {'error': ret}
return {'error': e.response['Error']['Code']}
except Exception as e:
return {'error': e}
try:
filename = '{}_{}'.format(batch_id, url_hash)
batch_key = batch_id.rsplit('-', 1)[0]
ret = put_cache(batch_key, filename, content)
if 'error' in ret and ret['error'] == 'NoSuchBucket':
ret = put_cache(batch_key, filename, content)
if 'error' in ret:
return {'success': False, 'error': ret['error']}
_groups = str(map(string.strip,
groups.split(',')))[1:-1].replace('\'', '"')
sql1 = (
"insert into accessed (batch_id, groups, status, b64url, url_hash) "
"values ('{}', '{{{}}}', '{}', '{}', '{}');".format(
batch_id, _groups, 0, b64url, url_hash))
sql2 = ("insert into cached (b64url, url_hash) values ('{}', '{}')"
";".format(b64url, url_hash))
dbwrapper.execute(sql1)
dbwrapper.execute(sql2)
now = datetime.now()
log_line = json.dumps({
'date': str(now),
'batch_id': batch_id,
'groups': groups,
'url': base64.urlsafe_b64decode(b64url),
})
cachelog.get_logger(batch_id, now.strftime('%Y%m%d'),
FSCACHEDIR).info(log_line)
except Exception as e:
return {'success': False, 'error': e}
return {'success': True}
def ufile_set_cache(b64url,
url_hash,
batch_id,
groups,
content,
refresh=False):
if not hasattr(ufile_set_cache, '_auth'):
put_auth = putufile.PutUFile(public_key, private_key)
setattr(ufile_set_cache, '_auth', put_auth)
try:
sio = StringIO(content)
filename = '{}_{}'.format(batch_id, url_hash)
batch_key = batch_id.rsplit('-', 1)[0]
ret, resp = ufile_set_cache._auth.putstream(batch_key, filename, sio)
if resp.status_code != 200:
if resp.status_code == 400:
if json.loads(resp.content)[u'ErrMsg'] == u'bucket not exist':
ensure_bucket(batch_key)
raise Exception(
'{} bucket not exist, create, upload again'.format(
batch_key))
raise Exception('{} upload ufile error: {}'.format(
batch_id, b64url))
_groups = str(map(string.strip,
groups.split(',')))[1:-1].replace('\'', '"')
sql1 = (
"insert into accessed (batch_id, groups, status, b64url, url_hash) "
"values ('{}', '{{{}}}', '{}', '{}', '{}');".format(
batch_id, _groups, 0, b64url, url_hash))
sql2 = ("insert into cached (b64url, url_hash) values ('{}', '{}')"
";".format(b64url, url_hash))
dbwrapper.execute(sql1)
dbwrapper.execute(sql2)
now = datetime.now()
log_line = json.dumps({
'date': str(now),
'batch_id': batch_id,
'groups': groups,
'url': base64.urlsafe_b64decode(b64url),
})
cachelog.get_logger(batch_id, now.strftime('%Y%m%d'),
FSCACHEDIR).info(log_line)
except Exception as e:
return {'success': False, 'error': e}
return {'success': True}
def process(url, batch_id, parameter, manager, *args, **kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader',
DownloadWrapper('s3', headers, REGION_NAME))
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout = int(timeout)
today_str = datetime.now().strftime('%Y%m%d')
get_logger(batch_id, today_str, '/opt/service/log/').info('start download')
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout,
encoding='gb18030')
if kwargs and kwargs.get("debug"):
print(len(content), "\n", content[:1000])
if content is False:
return False
content_urls = []
get_logger(batch_id, today_str, '/opt/service/log/').info('start parsing')
tree = lxml.html.fromstring(content)
urls = tree.xpath('//td[@class="title"]/a/@href')
if urls == []:
get_logger(batch_id, today_str,
'/opt/service/log/').info('start download2')
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout,
encoding='gb18030',
refresh=True)
if content is False:
return False
tree = lxml.html.fromstring(content)
urls = tree.xpath('//td[@class="title"]/a/@href')
for url in urls:
content_urls.append(urlparse.urljoin('http://data.eastmoney.com/',
url))
get_logger(batch_id, today_str,
'/opt/service/log/').info('start put content')
manager.put_urls_enqueue('dongcaigonggao-content-20160620', content_urls)
return True
def process(url, batch_id, parameter, manager, *args, **kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader',
DownloadWrapper(None, headers, REGION_NAME))
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CacheS3(head + '-json-' + tail))
if not hasattr(process, '_regs'):
setattr(process, '_regs',
re.compile(urlparse.urljoin(SITE, 'search\?word=(.+)')))
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout = int(timeout)
today_str = datetime.now().strftime('%Y%m%d')
word = urllib.unquote(process._regs.match(url).group(1))
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str,
'/opt/service/log/').info('start download')
refresh = False
for _ in range(5):
try:
content = process._downloader.downloader_wrapper(
url,
batch_id,
gap,
timeout=timeout,
encoding='gb18030',
refresh=refresh)
if content == '':
return False
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str,
'/opt/service/log/').info('start parsing url')
result = parse_search_json_v0707(content, word)
break
except:
refresh = True
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str,
'/opt/service/log/').info('start post json')
return process._cache.post(url, json.dumps(result))
def process(url, batch_id, parameter, manager, *args, **kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {"Host": domain_name}
setattr(process, '_downloader', DownloadWrapper('s3', headers, REGION_NAME))
if not hasattr(process, '_cache'):
setattr(process, '_cache', CacheS3(batch_id.split('-', 1)[0]+'-json'))
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout= int(timeout)
today_str = datetime.now().strftime('%Y%m%d')
get_logger(batch_id, today_str, '/opt/service/log/').info('start download content')
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout,
encoding='gb18030')
if kwargs and kwargs.get("debug"):
print(len(content), "\n", content[:1000])
if content is False:
return False
get_logger(batch_id, today_str, '/opt/service/log/').info('start parsing content')
begin = content.find('<div class="mainbox">')
end = content.find('<div id="footer">', begin)
tree = lxml.html.fromstring(content[begin:end])
title = tree.xpath('//div[@class="content"]/h4/text()')
if isinstance(title, list) and len(title) > 0:
title = title[0]
else:
title = None
public_date = tree.xpath('//div[@class="content"]/h5/text()')
if isinstance(public_date, list) and len(public_date) > 0:
public_date = public_date[0]
else:
public_date = None
body = tree.xpath('//div[@class="content"]//pre/text()')
if isinstance(body, list) and len(body) > 0:
body = body[0]
else:
body = None
notice_content = json.dumps({'url': url, 'title': title, 'public_date': public_date, 'body': body})
get_logger(batch_id, today_str, '/opt/service/log/').info('start post json')
ret = process._cache.post(url, notice_content)
return ret
def work(self, batch_id, queue_dict, url_id, other_batch_process_time, *args, **kwargs):
try:
batch_key_filename = batch_id.rsplit('-', 1)[0].replace('-', '_')
module = __import__('workers.{}'.format(batch_key_filename), fromlist=['process'])
except:
module = __import__('workers.prefetch', fromlist=['process'])
today_str = datetime.now().strftime('%Y%m%d')
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str, '/opt/service/log/').info('begin get url from thinhash redis')
# TODO change to hmget
url = queue_dict['thinhash'].hget(url_id)
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str, '/opt/service/log/').info('end get url from thinhash redis')
try:
process_status = module.process(url,
batch_id,
self._batch_param[batch_id],
self.manager,
other_batch_process_time,
*args,
**kwargs)
except Exception as e:
Record.instance().add_exception(batch_id, url, repr(e))
queue_dict['queue'].task_done(url_id)
return
if process_status:
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str, '/opt/service/log/').info('begin task done for record redis')
queue_dict['queue'].task_done(url_id)
Record.instance().increase_success(batch_id)
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str, '/opt/service/log/').info('end task done for record redis')
def process(url, batch_id, parameter, manager, other_batch_process_time, *args,
**kwargs):
# 药材的详情页涉及2个部分：价格历史history和边栏sidebar，以下的ytw/second/是价格历史的url，返回一个大的json；
# 所以在最后处理的时候还要额外向另一个url发送一次请求，以获得边栏信息,由于要储存到同一个result.json中，因此不再放入队列，而是直接在process里完成
today_str = datetime.now().strftime('%Y%m%d')
get_logger(batch_id, today_str,
'/opt/service/log/').info('process {}'.format(url))
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_regs'):
setattr(
process,
'_regs',
{
'home':
re.compile('http://www.yt1998.com/variteyIndexInfo.html'),
'kind':
re.compile('http://www.yt1998.com/issueIndexInfo.html\?code='),
'history':
re.compile(
'http://www.yt1998.com/ytw/second/indexMgr/getIndexInfo.jsp\?code=(\d+)&type=1&varitey_name=(.*)'
) #这是价格历史的url
})
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
if not hasattr(process, '_next_patterns'):
setattr(
process,
'_next_patterns',
{
'home':
'http://www.yt1998.com/issueIndexInfo.html?code={}', #the format of kind
'kind':
'http://www.yt1998.com/ytw/second/indexMgr/getIndexInfo.jsp?code={}&type=1&varitey_name={}', #the format of history
'history':
'http://www.yt1998.com/variteyIndexInfo.html?varitey_code={}' #the format of sidebar
})
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout = int(timeout)
gap = max(gap - other_batch_process_time, 0)
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
get_logger(batch_id, today_str,
'/opt/service/log/').info('label : {}'.format(label))
if label in [
'home', 'kind'
]: #I handle home-page and kind-page in one code block cuz they are in same web format
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout,
encoding='utf-8',
refresh=True)
dom = lxml.html.fromstring(content)
dd_labels = dom.xpath('//dd')
urls = []
for single_dd in dd_labels:
rels = single_dd.xpath('.//@rel')
if not rels:
get_logger(batch_id, today_str, '/opt/service/log/').info(
'wrong rels content : {}'.format(rels))
continue
for rel in rels:
code = rel.split(
','
)[-2] #在home页面，rel的格式为 'Z,家种类' code为Z；在kind页面，rel格式为'Z,家种类,000001,枸杞'，code为000001
if label == 'home':
urls.append(process._next_patterns[label].format(code))
else: # label == 'kind'
name = str(rel.split(',')[-1])
urls.append(process._next_patterns[label].format(
code, urllib.quote(name)))
manager.put_urls_enqueue(batch_id, urls)
elif label == 'history': #开始提取单种药品数据
code = m.group(1)
name = urllib.unquote(m.group(2))
sidebar_url = process._next_patterns[label].format(code)
sidebar_content = process._downloader.downloader_wrapper(
sidebar_url,
batch_id,
gap,
timeout=timeout,
encoding='utf-8',
refresh=True)
sidebar_dom = lxml.html.fromstring(sidebar_content)
sidebar_label = sidebar_dom.xpath(
'//div[@class="box-con-r fr"]/table//tr')
if not isinstance(sidebar_label, list) or len(sidebar_label) != 19:
get_logger(batch_id, today_str,
'/opt/service/log/').info('not legal list!')
return False
sidebar_item = {} # 边栏信息
for index in range(1, 16):
line_content = sidebar_label[index].xpath(
'./td/text()') #line content格式为 权重比：0.0278、市净率：2.00...
parts = line_content[0].split(
'：') # chinese colon :left part as key,right part as value
sidebar_item[parts[0]] = parts[1]
line_content = sidebar_label[16].xpath(
'./th/text()') #最后更新时间的样式与其他不同，为th
parts = line_content[0].split('：')
sidebar_item[parts[0]] = parts[1]
history_content = process._downloader.downloader_wrapper(
url,
batch_id,
gap,
timeout=timeout,
encoding='utf-8',
refresh=True)
if history_content == '':
return False
get_logger(
batch_id, today_str,
'/opt/service/log/').info('history downloading finished')
history_item = json.loads(history_content)[
u'DayMonthData'] #从结果中提取每天数据
price_history = {} #价格历史
for raw_daily_data in history_item:
date = raw_daily_data[u'Date_time']
price = raw_daily_data[u'DayCapilization']
price_history[date] = price
result_item = {}
result_item['name'] = name
result_item['info'] = sidebar_item
result_item['price_history'] = price_history
result_item['source'] = sidebar_url
return process._cache.post(url,
json.dumps(result_item,
ensure_ascii=False),
refresh=True)
def process(url, batch_id, parameter, manager, other_batch_process_time, *args,
**kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
cookie = kwargs.get('cookie', None)
# cookie = "gr_user_id=0fceb70d-e0ab-4c16-8f21-d49b5d242b0e; PHPSESSID=ltro2cjbvonlg6mu4hupe7dcv1; CNZZDATA1254842228=371101890-1469690209-null%7C1472547698"
if cookie:
headers.update({'Cookie': cookie})
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_cache'):
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
if not hasattr(process, '_regs'):
setattr(
process, '_regs', {
'search':
re.compile(
urlparse.urljoin(SITE,
'search\?key=(.+?)&index=(\d+)&p=(\d+)')),
'detail':
re.compile(
urlparse.urljoin(
SITE,
'company_getinfos\?unique=(.+?)&companyname=(.+?)&tab=base'
)),
'invest':
re.compile(
urlparse.urljoin(
SITE,
'company_getinfos\?unique=(.+?)&companyname=(.+?)(?:&p=(\d+))?&tab=touzi(?:&box=touzi)?'
)),
})
method, gap, js, timeout, data = parameter.split(':')
gap = float(max(0, float(gap) - other_batch_process_time))
timeout = int(timeout)
today_str = datetime.now().strftime('%Y%m%d')
# if kwargs and kwargs.get("debug"):
# get_logger(batch_id, today_str, '/opt/service/log/').info('start download')
def reformat(info): # 将info按企查查页面顺序插入队列
temp = info['info']
del info['info']
info['info'] = []
info['info'].append(("统一社会信用码", temp['unified_social_credit_code']))
info['info'].append(("注册号", temp['registration_id']))
info['info'].append(("组织机构代码", temp['organization_code']))
info['info'].append(("经营状态", temp['status']))
info['info'].append(("公司类型", temp['business_type']))
info['info'].append(("成立日期", temp['begin']))
info['info'].append(("法定代表", temp['legal_person']))
info['info'].append(("注册资本", temp['registered_capital']))
info['info'].append(("营业期限", temp['end']))
info['info'].append(("登记机关", temp['registration_authority']))
info['info'].append(("发照日期", temp['approval_date']))
info['info'].append(("企业地址", temp['address']))
info['info'].append(("经营范围", temp['business_scope']))
return info
def parse_company_investment(tree): # 解析对外投资页面，将子公司存入sub_companies字段下
invest_dict = {'sub_companies': []}
for sub_company in tree.cssselect('.list-group a.list-group-item'):
sub_name = sub_company.cssselect(
'span.clear .text-lg')[0].text_content().strip()
href = sub_company.get('href')
province, key_num = href.rsplit('_', 2)[-2:]
invest_dict['sub_companies'].append({
'name': sub_name,
'key_num': key_num,
'province': province,
'href': href,
})
return invest_dict
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
method,
timeout=timeout,
encoding='utf-8')
# print(url, file=log_file)
cookie = kwargs.get('cookie', None)
if not cookie:
get_logger(batch_id, today_str,
'/opt/service/log/').info("No cookie in worker")
return False
if content == '':
get_logger(batch_id, today_str, '/opt/service/log/').info("no content")
content = requests.get(url, cookies={1: cookie}).text
if content:
# print("got content", file=log_file)
get_logger(batch_id, today_str,
'/opt/service/log/').info('got content')
if not content and url.endswith("tab=touzi&box=touzi"):
get_logger(batch_id, today_str,
'/opt/service/log/').info("void invest page")
return True
invest_pat = "http://www.qichacha.com/company_getinfos?unique={key_num}&companyname={name}&p={p}&tab=touzi&box=touzi"
main_pat = "http://www.qichacha.com/company_getinfos?unique={key_num}&companyname={name}&tab=base"
search_pat = "http://www.qichacha.com/search?key={name}&index=0&p={p}"
parser = QiParser()
tree = lxml.html.fromstring(
content.replace('<em>', '').replace('</em>', ''))
# if kwargs and kwargs.get("debug"):
# print('start parsing url')
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
if label == 'search': # 搜索页面解析
comp_name = urllib.unquote(m.group(1))
dic = {'search_name': comp_name, 'names': []}
urls = []
if tree.cssselect('.table-search-list') and tree.cssselect(
'.tp2_tit a'):
items = tree.cssselect('.table-search-list')
for idx, i in enumerate(items):
if not i.xpath('.//*[@class=\"tp2_tit clear\"]/a/text()'):
continue
item = {}
item['name'] = i.xpath(
'.//*[@class=\"tp2_tit clear\"]/a/text()')[0]
# print(item['name'], file=log_file)
item['href'] = i.xpath(
'.//*[@class=\"tp2_tit clear\"]/a/@href')[0]
item['status'] = i.xpath(
'.//*[@class=\"tp5 text-center\"]/a/span/text()')[0]
item['key_num'] = item['href'].split('firm_')[1].split(
'.shtml')[0]
# print(item['key_num'], file=log_file)
if idx == 0 and comp_name == item[
'name']: # 若第一个搜索结果完全匹配则只添加第一个结果入待爬取队列
# get_logger(batch_id, today_str, '/opt/service/log/').info('appending', item['name'])
urls.append(
main_pat.format(key_num=item['key_num'],
name=item['name']))
urls.append(
invest_pat.format(key_num=item['key_num'],
name=item['name'],
p='1'))
break
elif idx < 3: # 如果第一个不完全匹配， 将前三个搜索结果加入待爬取队列
urls.append(
main_pat.format(key_num=item['key_num'],
name=item['name']))
urls.append(
invest_pat.format(key_num=item['key_num'],
name=item['name'],
p='1'))
dic['names'].append(item['name'])
if not urls:
return True
manager.put_urls_enqueue(batch_id, urls)
if not dic['names']:
return True
else: # 不完全匹配时将search_name与前三个搜索结果存入json用作别名映射
data = json.dumps(dic, encoding='utf-8', ensure_ascii=False)
return process._cache.post(url, data)
elif label == 'detail': # 解析详情页面
comp_name = urllib.unquote(m.group(2))
# print(comp_name, 'detail', file=log_file)
all_info = parser.parse_detail(tree)
all_info['name'] = comp_name
all_info['source'] = url
all_info['access_time'] = datetime.utcnow().isoformat()
all_info = parser.parser_patch(tree, all_info)
all_info = reformat(all_info)
data = json.dumps(all_info, encoding='utf-8', ensure_ascii=False)
get_logger(batch_id, today_str, '/opt/service/log/').info(data)
if not any([i[1] for i in all_info['info']]):
return False
return process._cache.post(url, data)
else: # 解析投资页面
comp_name = urllib.unquote(m.group(2))
key_num = m.group(1)
page = int(m.group(3))
pages = tree.xpath(".//a[@id=\"ajaxpage\"]/text()")
if '>' in pages:
urls = [
invest_pat.format(key_num=key_num,
name=comp_name,
p=str(page + 1))
]
manager.put_urls_enqueue(batch_id, urls)
invest_dict = parse_company_investment(tree)
# print(invest_dict, file=log_file)
if not invest_dict['sub_companies']:
return True
invest_dict['name'] = comp_name
invest_dict['source'] = url
invest_dict['access_time'] = datetime.utcnow().isoformat()
data = json.dumps(invest_dict,
encoding='utf-8',
ensure_ascii=False)
get_logger(batch_id, today_str, '/opt/service/log/').info(data)
return process._cache.post(url, data)
def process(url, batch_id, parameter, manager, other_batch_process_time, *args, **kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
if not hasattr(process, '_regs'):
setattr(process, '_regs', {
'main': re.compile(r'http://china.chemnet.com/hot-product/(\w|\d+).html'),
'prd': re.compile(r'http://china.chemnet.com/product/pclist--(.+?)--0.html'),
'comps': re.compile(r'http://china.chemnet.com/product/search.cgi')
})
def safe_state(statement):
return statement[0] if statement else ''
def xpath_string(n):
return "//*[@id=\"main\"]/div[1]/div[1]/table/tr[" + str(n) + "]/td[2]/text()"
method, gap, js, timeout, data = parameter.split(':')
gap = float(max(0, float(gap) - other_batch_process_time))
timeout= int(timeout)
compspat = 'http://china.chemnet.com/product/search.cgi?skey={};use_cas=0;f=pclist;p={}'
today_str = datetime.now().strftime('%Y%m%d')
# if kwargs and kwargs.get("debug"):
# get_logger(batch_id, today_str, '/opt/service/log/').info('start download')
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout,
# encoding='gb18030',
refresh=True
)
# print(content)
if content == '':
get_logger(batch_id, today_str, '/opt/service/log/').info(url + ' no content')
return False
# if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str, '/opt/service/log/').info('start parsing url')
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
page = etree.HTML(content.replace('<sub>', '').replace('</sub>', ''))
if label == 'main':
# print("add chems")
chems = page.xpath("//*[@id=\"main\"]/div[1]/div[2]/dl/dd/ul/li/p[2]/a/@href") # links for chems in main page
chems = [ urlparse.urljoin(SITE, chem) for chem in chems]
get_logger(batch_id, today_str, '/opt/service/log/').info('adding chems urls into queue')
manager.put_urls_enqueue(batch_id, chems)
return True
elif label == 'prd':
chem_uri = m.group(1)
chem_name = page.xpath("//*[@id=\"main\"]/div[1]/div[1]/table/tr[1]/td[2]/text()")[0]
get_logger(batch_id, today_str, '/opt/service/log/').info(chem_name + " main page")
comps = page.xpath("//*[@id=\"main\"]/div[2]/div[2]/dl/dd/form/table/tr[1]/td[2]/a[1]")
pagetext = page.xpath("//*[@id=\"main\"]/div[2]/div[2]/dl/dd/h6/div/text()[1]")
# print(pagetext[0])
total = int(re.compile(r'共有(\d+)条记录').search(pagetext[0].encode('utf-8')).group(1))
total = total // 10 + 1 if total % 10 != 0 else total // 10
dic = {
u'source': url,
u'中文名称': page.xpath(xpath_string(1))[0] if page.xpath(xpath_string(1)) else '',
u'英文名称': page.xpath(xpath_string(2))[0] if page.xpath(xpath_string(2)) else '',
u'中文别名': page.xpath(xpath_string(3))[0] if page.xpath(xpath_string(3)) else '',
u'CAS_RN': page.xpath(xpath_string(4))[0] if page.xpath(xpath_string(4)) else '',
u'EINECS': page.xpath(xpath_string(5))[0] if page.xpath(xpath_string(5)) else '',
u'分子式': page.xpath(xpath_string(6))[0] if page.xpath(xpath_string(6)) else '',
u'分子量': page.xpath(xpath_string(7))[0] if page.xpath(xpath_string(7)) else '',
u'危险品标志': page.xpath(xpath_string(8))[0].strip() if page.xpath(xpath_string(8)) else '',
u'风险术语': page.xpath(xpath_string(9))[0].strip() if page.xpath(xpath_string(9)) else '',
u'安全术语': page.xpath(xpath_string(10))[0].strip() if page.xpath(xpath_string(10)) else '',
u'物化性质': page.xpath("//*[@id=\"main\"]/div[1]/div[1]/table/tr[11]/td[2]/p/text()") if page.xpath("//*[@id=\"main\"]/div[1]/div[1]/table/tr[11]/td[2]/p/text()") else [],
u'用途': page.xpath(xpath_string(12))[0] if page.xpath(xpath_string(12)) else '',
u'上游原料': page.xpath('//*[@id=\"main\"]/div[1]/div[1]/table/tr[14]/td[2]/a/text()') if page.xpath('//*[@id=\"main\"]/div[1]/div[1]/table/tr[14]/td[2]/a/text()') else [],
u'下游产品': page.xpath('//*[@id=\"main\"]/div[1]/div[1]/table/tr[15]/td[2]/a/text()') if page.xpath('//*[@id=\"main\"]/div[1]/div[1]/table/tr[15]/td[2]/a/text()') else [],
}
data = json.dumps(dic, encoding='utf-8', ensure_ascii=False)
new_urls = []
for t in range(total):
new_url = compspat.format(chem_uri, str(t))
get_logger(batch_id, today_str, '/opt/service/log/').info("new url" + new_url)
new_urls.append(new_url)
manager.put_urls_enqueue(batch_id, new_urls)
get_logger(batch_id, today_str, '/opt/service/log/').info('start posting prd page to cache')
return process._cache.post(url, data)
else:
chem_name = page.xpath("//*[@id=\"main\"]/div[1]/div[1]/table/tr[1]/td[2]/text()")[0]
total = len(page.xpath("//*[@id=\"main\"]/div[2]/div[2]/dl/dd/form")) # total num of suppliers
dic = ''
for i in range(1, total + 1):
c = safe_state(page.xpath("//*[@id=\"main\"]/div[2]/div[2]/dl/dd/form[{}]".format(str(i))))
if c is '':
break
comp = {}
comp[u'source'] = url
comp[u'chem_name'] = chem_name
comp[u'name'] = safe_state(c.xpath(".//table/tr[1]/td[2]/a[1]/text()"))
comp[u'tel'] = safe_state(c.xpath(".//table/tr[2]/td[2]/text()"))
comp[u'fax'] = safe_state(c.xpath(".//table/tr[3]/td[2]/text()"))
comp[u'website'] = safe_state(c.xpath(".//table/tr[4]/td[2]/a/text()"))
dic += json.dumps(comp, encoding='utf-8', ensure_ascii=False) + '\n'
dic = dic.strip()
get_logger(batch_id, today_str, '/opt/service/log/').info('start posting companies to cache')
return process._cache.post(url, dic)
def process(url, batch_id, parameter, manager, other_batch_process_time, *args,
**kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
if not hasattr(process, '_regs'):
setattr(
process, '_regs', {
'main':
re.compile(
r'http://www.sge.com.cn/xqzx/mrxq/index_(\d+).shtml'),
'info':
re.compile(r'http://www.sge.com.cn/xqzx/mrxq/(\d+).shtml')
})
method, gap, js, timeout, data = parameter.split(':')
gap = float(max(0, float(gap) - other_batch_process_time))
timeout = int(timeout)
today_str = datetime.now().strftime('%Y%m%d')
# print(url)
# if kwargs and kwargs.get("debug"):
# get_logger(batch_id, today_str, '/opt/service/log/').info('start download')
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout)
# print(content)
if content == '':
get_logger(batch_id, today_str,
'/opt/service/log/').info(url + ' no content')
return False
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
# print("not match")
continue
page = etree.HTML(content)
if label == 'main':
get_logger(batch_id, today_str,
'/opt/service/log/').info('in list page')
urls = page.xpath(".//ul[@id='zl_list']/li/a/@href")
urls = [urlparse.urljoin(SITE, list_url) for list_url in urls]
get_logger(batch_id, today_str,
'/opt/service/log/').info(str(urls))
# get_logger(batch_id, today_str, '/opt/service/log/').info('||'.join(prd_links) + ' added to queue')
manager.put_urls_enqueue(batch_id, urls)
return True
elif label == 'info':
dic = {}
date = page.xpath(".//h5[@class='con_h5']/text()")[0].split(
u'\xa0')[0]
header = page.xpath(
".//div[@id='page_con']/table/tbody/tr[1]/td//text()")
infos = page.xpath(
".//div[@id='page_con']/table/tbody/tr/td[1]//text()")
infos = [info.strip() for info in infos if info.strip()]
idx = -1
for index, prod in enumerate(list(infos)):
if u"Pt9995" in prod:
idx = str(index + 1)
break
if idx == -1:
return True
pt_infos = page.xpath(
".//div[@id='page_con']/table/tbody/tr[{}]/td//text()".format(
idx))
if not pt_infos:
get_logger(
batch_id, today_str,
'/opt/service/log/').info("No pt info on this page " + url)
return True
for col, value in zip(header, pt_infos):
dic[col] = value.strip()
dic[u'日期'] = date
dic[u'source'] = url
dic[u'access_time'] = datetime.utcnow().isoformat()
data = json.dumps(dic, ensure_ascii=False)
get_logger(batch_id, today_str, '/opt/service/log/').info(data)
return process._cache.post(url, data)
def process(url, batch_id, parameter, manager, other_batch_process_time, *args,
**kwargs):
# 药材的详情页涉及2个部分：价格历史history和边栏sidebar，以下的ytw/second/是价格历史的url，返回一个大的json；
# 所以在最后处理的时候还要额外向另一个url发送一次请求，以获得边栏信息,由于要储存到同一个result.json中，因此不再放入队列，而是直接在process里完成
today_str = datetime.now().strftime('%Y%m%d')
get_logger(batch_id, today_str,
'/opt/service/log/').info('process {}'.format(url))
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_regs'):
setattr(
process, '_regs', {
'list_view':
re.compile(
'http://www.yt1998.com/price/nowDayPriceQ\!getPriceList.do\?pageIndex=(\d+)&pageSize=(\d+)'
),
'detail_view':
re.compile(
'http://www.yt1998.com/ytw/second/priceInMarket/getPriceHistory.jsp\?ycnam=(.*)&guige=(.*)&chandi=(.*)&market=(.*)'
)
})
if not hasattr(process, '_sellerMarket_list'):
setattr(process, '_sellerMarket_list',
['', u'亳州市场', u'安国市场', u'玉林市场', u'成都市场'])
# http://www.yt1998.com/price/nowDayPriceQ!getPriceList.do?pageIndex=0&pageSize=500
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout = int(timeout)
gap = max(gap - other_batch_process_time, 0)
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
get_logger(batch_id, today_str,
'/opt/service/log/').info('label : {}'.format(label))
if label == 'list_view':
get_logger(batch_id, today_str, '/opt/service/log/').info(label)
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout,
encoding='utf-8',
refresh=True)
get_logger(batch_id, today_str,
'/opt/service/log/').info('download ok')
get_logger(batch_id, today_str,
'/opt/service/log/').info(len(content))
list_item = json.loads(content)
urls = []
for detail_item in list_item[u'data']:
detail_url_pattern = 'http://www.yt1998.com/ytw/second/priceInMarket/getPriceHistory.jsp?ycnam={}&guige={}&chandi={}&market={}'
ycnam = str(detail_item[u'ycnam'])
chandi = str(detail_item[u'chandi'])
market = str(detail_item[u'market'])
guige = str(detail_item[u'guige'])
detail_url = detail_url_pattern.format(urllib.quote(ycnam),
urllib.quote(guige),
urllib.quote(chandi),
urllib.quote(market))
urls.append(detail_url)
get_logger(batch_id, today_str,
'/opt/service/log/').info('len urls')
get_logger(batch_id, today_str,
'/opt/service/log/').info(len(urls))
manager.put_urls_enqueue(batch_id, urls)
total_num = int(list_item[u'total'])
pageIndex = int(m.group(1))
pageSize = int(m.group(2))
if pageIndex == 0:
print(total_num // pageSize)
for index in range(1, total_num // pageSize + 1):
get_logger(batch_id, today_str,
'/opt/service/log/').info('iiiiiindex')
get_logger(batch_id, today_str,
'/opt/service/log/').info(index)
list_pattern = 'http://www.yt1998.com/price/nowDayPriceQ!getPriceList.do?pageIndex={}&pageSize={}'
list_url = list_pattern.format(index, pageSize)
manager.put_urls_enqueue(batch_id, [list_url])
return True
elif label == 'detail_view':
get_logger(batch_id, today_str, '/opt/service/log/').info(label)
ycnam = urllib.unquote(m.group(1))
guige = urllib.unquote(m.group(2))
chandi = urllib.unquote(m.group(3))
market = urllib.unquote(m.group(4))
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout,
encoding='utf-8',
refresh=True)
get_logger(batch_id, today_str,
'/opt/service/log/').info(len(content))
history_item = json.loads(content)
get_logger(batch_id, today_str,
'/opt/service/log/').info('downloaded')
price_history = {}
for raw_daily_data in history_item[u'DayPriceData']:
date = raw_daily_data[u'Date_time']
price = raw_daily_data[u'DayCapilization']
price_history[date] = price
source_url = 'http://www.yt1998.com/priceHistory.html?keywords={}&guige={}&chandi={}&market={}'
get_logger(batch_id, today_str, '/opt/service/log/').info('source')
get_logger(batch_id, today_str, '/opt/service/log/').info(
len(process._sellerMarket_list))
result_item = {
'name': ycnam,
'productGrade': guige,
'productPlaceOfOrigin': chandi,
'sellerMarket': process._sellerMarket_list[int(market)],
'price_history': price_history,
'source': source_url.format(ycnam, guige, chandi, market),
}
print(result_item)
result_item['access_time'] = datetime.utcnow().isoformat(
) # 从上面source的赋值可看出每个item都对应不同的参数
return process._cache.post(url,
json.dumps(result_item,
ensure_ascii=False),
refresh=True)
def process(url, batch_id, parameter, manager, other_batch_process_time, *args,
**kwargs):
home_page = 'http://app1.sfda.gov.cn/datasearch/face3/base.jsp?tableId=25&tableName=TABLE25&title=%B9%FA%B2%FA%D2%A9%C6%B7&bcId=124356560303886909015737447882'
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_reg'):
setattr(
process, '_reg', {
'detail':
re.compile(
'http://app1.sfda.gov.cn/datasearch/face3/content.jsp\?tableId=25&tableName=TABLE25&tableView=%B9%FA%B2%FA%D2%A9%C6%B7&Id=(\d+)'
),
})
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout = int(timeout)
gap = max(gap - other_batch_process_time, 0)
today_str = datetime.now().strftime('%Y%m%d')
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str,
'/opt/service/log/').info('start download')
data = {
'tableId': '25',
'State': '1',
'bcId': '124356560303886909015737447882',
'State': '1',
'curstart': 1, #here!
'State': '1',
'tableName': 'TABLE25',
'State': '1',
'viewtitleName': 'COLUMN167',
'State': '1',
'viewsubTitleName': 'COLUMN166,COLUMN170,COLUMN821',
'State': '1',
'tableView': '%E5%9B%BD%E4%BA%A7%E8%8D%AF%E5%93%81',
'State': '1',
}
if url == home_page:
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str,
'/opt/service/log/').info('start parsing url')
page = 1
while 1:
data['curstart'] = page
content = process._downloader.downloader_wrapper(
'http://app1.sfda.gov.cn/datasearch/face3/search.jsp',
batch_id,
gap,
method='post',
timeout=timeout,
refresh=True,
data=data)
# if page == 3:
# return
ids = re.findall(u'国产药品&Id=(\d+)', content)
if not ids:
break
url_pattern = 'http://app1.sfda.gov.cn/datasearch/face3/content.jsp?tableId=25&tableName=TABLE25&tableView=%B9%FA%B2%FA%D2%A9%C6%B7&Id={}'
urls = []
for drug_id in ids:
url = url_pattern.format(drug_id)
urls.append(url)
manager.put_urls_enqueue(batch_id, urls)
page += 1
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str, '/opt/service/log/').info(
'going to page{}'.format(page))
return
elif process._reg['detail'].match(url):
content = process._downloader.downloader_wrapper(
url,
batch_id,
gap,
timeout=timeout,
)
if content == '':
return False
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str,
'/opt/service/log/').info('start parsing url')
dom = lxml.html.fromstring(content)
table = dom.xpath('//tr')
item = {
'license_number':
table[1].xpath('./td')[1].xpath('./text()'), #[u'批准文号'],
'product_name_chs':
table[2].xpath('./td')[1].xpath('./text()'), #[u'产品名称'],
'product_name_eng':
table[3].xpath('./td')[1].xpath('./text()'), #[u'英文名称'],
'commodity_name_chs':
table[4].xpath('./td')[1].xpath('./text()'), #[u'商品名'],
'drug_form':
table[5].xpath('./td')[1].xpath('./text()'), #[u'剂型'],
'specification':
table[6].xpath('./td')[1].xpath('./text()'), #[u'规格'],
'manufacturer_chs':
table[7].xpath('./td')[1].xpath('./text()'), #[u'生产单位'],
'manuf_address_chs':
table[8].xpath('./td')[1].xpath('./text()'), #[u'生产地址'],
'category':
table[9].xpath('./td')[1].xpath('./text()'), #[u'产品类别'],
'license_data':
table[11].xpath('./td')[1].xpath('./text()'), #[u'批准日期'],
'standard_code':
table[12].xpath('./td')[1].xpath('./text()'), #[u'药品本位码'],
'standard_code_remark':
table[13].xpath('./td')[1].xpath('./text()'), #[u'药品本位码备注'],
'source': [url],
}
for k, v in item.iteritems():
if len(v) > 0:
item[k] = v[0]
else:
item[k] = None
return process._cache.post(url, json.dumps(item, ensure_ascii=False))
def process(url, batch_id, parameter, manager, other_batch_process_time, *args,
**kwargs):
today_str = datetime.now().strftime('%Y%m%d')
get_logger(batch_id, today_str,
'/opt/service/log/').info('process {}'.format(url))
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_regs'):
setattr(
process, '_regs', {
'list_view':
re.compile(
'http://www.yt1998.com/price/nowDayPriceQ\!getPriceList.do\?pageIndex=(\d+)&pageSize=(\d+)'
),
'detail_view':
re.compile(
'http://www.yt1998.com/ytw/second/priceInMarket/getPriceHistory.jsp\?ycnam=(.*)&guige=(.*)&chandi=(.*)&market=(.*)'
)
})
# http://www.yt1998.com/price/nowDayPriceQ!getPriceList.do?pageIndex=0&pageSize=20
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
method, gap, js, timeout, data = parameter.split(':')
gap = float(max(0, float(gap) - other_batch_process_time))
timeout = int(timeout)
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
get_logger(batch_id, today_str,
'/opt/service/log/').info('label : {}'.format(label))
if label == 'list_view':
get_logger(batch_id, today_str, '/opt/service/log/').info(label)
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout,
encoding='utf-8',
refresh=True)
get_logger(batch_id, today_str,
'/opt/service/log/').info('download ok')
list_item = json.loads(content)
for detail_item in list_item[u'data']:
detail_item[u'access_time'] = datetime.utcnow().isoformat()
total_num = int(list_item[u'total'])
pageIndex = int(m.group(1))
pageSize = int(m.group(2))
if pageIndex == 0:
for index in range(1, total_num // pageSize + 1):
get_logger(batch_id, today_str,
'/opt/service/log/').info('index:')
get_logger(batch_id, today_str,
'/opt/service/log/').info(index)
list_pattern = 'http://www.yt1998.com/price/nowDayPriceQ!getPriceList.do?pageIndex={}&pageSize={}'
list_url = list_pattern.format(index, pageSize)
manager.put_urls_enqueue(batch_id, [list_url])
return process._cache.post(url,
json.dumps(list_item,
ensure_ascii=False),
refresh=True)
def process(url, batch_id, parameter, manager, *args, **kwargs):
# 药材的详情页涉及2个部分：价格历史history和边栏sidebar，以下的ytw/second/是价格历史的url，返回一个大的json；
# 所以在最后处理的时候还要额外向另一个url发送一次请求，以获得边栏信息,由于要储存到同一个result.json中，因此不再放入队列，而是直接在process里完成
today_str = datetime.now().strftime('%Y%m%d')
get_logger(batch_id, today_str,
'/opt/service/log/').info('process {}'.format(url))
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader',
DownloadWrapper(None, headers, REGION_NAME))
if not hasattr(process, '_regs'):
setattr(
process,
'_regs',
{
'home':
re.compile('http://www.yt1998.com/variteyIndexInfo.html'),
'kind':
re.compile('http://www.yt1998.com/issueIndexInfo.html\?code='),
'history':
re.compile(
'http://www.yt1998.com/ytw/second/indexMgr/getIndexInfo.jsp\?code=(\d+)&type=1&varitey_name=(.*)'
) #这是价格历史的url
})
# if not hasattr(process, '_cache'):
# head, tail = batch_id.split('-')
# setattr(process, '_cache', CacheS3(head + '-json-' + tail))
if not hasattr(process, '_next_patterns'):
setattr(
process,
'_next_patterns',
{
'home':
'http://www.yt1998.com/issueIndexInfo.html?code={}', #the format of kind
'kind':
'http://www.yt1998.com/ytw/second/indexMgr/getIndexInfo.jsp?code={}&type=1&varitey_name={}', #the format of history
'history':
'http://www.yt1998.com/variteyIndexInfo.html?varitey_code={}' #the format of sidebar
})
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout = int(timeout)
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
get_logger(batch_id, today_str,
'/opt/service/log/').info('label : {}'.format(label))
if label in [
'home', 'kind'
]: #I handle home-page and kind-page in one code block cuz they are in same web format
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout,
encoding='utf-8',
refresh=True)
dom = lxml.html.fromstring(content)
dd_labels = dom.xpath('//dd')
urls = []
for single_dd in dd_labels:
rels = single_dd.xpath('.//@rel')
if not rels:
get_logger(batch_id, today_str, '/opt/service/log/').info(
'wrong rels content : {}'.format(rels))
continue
for rel in rels:
code = rel.split(
','
)[-2] #在home页面，rel的格式为 'Z,家种类' code为Z；在kind页面，rel格式为'Z,家种类,000001,枸杞'，code为000001
if label == 'home':
urls.append(process._next_patterns[label].format(code))
else: # label == 'kind'
name = str(rel.split(',')[-1])
urls.append(process._next_patterns[label].format(
code, urllib.quote(name)))
manager.put_urls_enqueue(batch_id, urls)
elif label == 'history': #开始提取单种药品数据
#由于之前的设计，传进来的是历史价格的url，在更新的时候已经用不到，但是为了尽量一致，减少变动，采用传入
#历史记录url，再提取其中的参数组成边栏url，发送请求得到当日价格的逻辑
code = m.group(1)
name = urllib.unquote(m.group(2))
sidebar_url = process._next_patterns[label].format(code)
sidebar_content = process._downloader.downloader_wrapper(
sidebar_url,
batch_id,
gap,
timeout=timeout,
encoding='utf-8',
refresh=True)
sidebar_dom = lxml.html.fromstring(sidebar_content)
sidebar_label = sidebar_dom.xpath(
'//div[@class="box-con-r fr"]/table//tr')
if not isinstance(sidebar_label, list) or len(sidebar_label) != 19:
get_logger(batch_id, today_str,
'/opt/service/log/').info('not legal list!')
return False
for index in range(1, 16):
line_content = sidebar_label[index].xpath(
'./td/text()') #line content格式为 权重比：0.0278、市净率：2.00...
parts = line_content[0].split(
'：') # chinese colon :left part as key,right part as value
if parts[0] == u'当前价格':
# print ('相等')
today_price = parts[1]
break
result_item = {}
result_item['today_price'] = today_price
result_item['name'] = name
result_item['url'] = sidebar_url
return True # 之后更改为新的cache
def process(url, batch_id, parameter, manager, other_batch_process_time, *args, **kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
if not hasattr(process, '_regs'):
setattr(process, '_regs', {
'main': re.compile(r'http://www.kmzyw.com.cn/bzjsp/biz_price_search/price_index_search.jsp'),
'prd': re.compile(r'http://www.kmzyw.com.cn/bzjsp/Biz_price_history/price_history_search.jsp\?name=(.*?)')
})
def timestamp2datetime(timestamp):
if isinstance(timestamp, (int, long, float)):
dt = datetime.utcfromtimestamp(timestamp)
else:
return "Not a valid timestamp"
mid = '-0' if dt.month < 10 else '-'
return str(dt.year) + mid + str(dt.month)
post_form = {
'pagecode': None,
# 'search_site': '%25E4%25BA%25B3%25E5%25B7%259E',
}
method, gap, js, timeout, data = parameter.split(':')
gap = float(max(0, float(gap) - other_batch_process_time))
timeout = int(timeout)
today_str = datetime.now().strftime('%Y%m%d')
get_logger(batch_id, today_str, '/opt/service/log/').info('start parsing url')
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
if label == 'main':
total_page = 10 # 初始化为一个较小的数，之后在获取页面内容后会更新此总页数
page_num = 1
while page_num < total_page + 1:
post_form['pagecode'] = page_num
# print(page_num)
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
method='post',
data=post_form,
timeout=timeout,
refresh=True
)
# print(content)
data = json.loads(content)
total_page = data['page'] # 从json中读出总页数
drugs = []
drug_url = 'http://www.kmzyw.com.cn/bzjsp/Biz_price_history/price_history_search.jsp?name={}'
for row in data['rows']:
# print(row['drug_name'])
drugs.append(drug_url.format(urllib.quote(str(row['drug_name'])).replace('%', '%25')))
manager.put_urls_enqueue(batch_id, drugs)
page_num += 1
return True
elif label == 'prd':
content = process._downloader.downloader_wrapper(
url,
batch_id,
gap,
timeout=timeout,
refresh=True
)
page = etree.HTML(content)
prd = page.xpath("/html/body/section[2]/h1/text()")[0]
idx = prd.index(u'品种')
prd = prd[:idx]
get_logger(batch_id, today_str, '/opt/service/log/').info(prd + " main page")
price_hist = page.xpath("/html/head/script[12]/text()")[0]
# print(price_hist)
data_pat = re.compile(r'series : \[(.*),marker')
m = data_pat.findall(price_hist)
dics = ''
if m:
# print(m[0])
data = m[0].split(',marker : { enabled : false ,radius : 3 } ,tooltip : { valueDecimals : 2 }},')
for d in data:
name = 'name'
data = 'data'
dic = eval(d + '}')
# print(dic)
cleaned = {}
cleaned['source'] = url
cleaned['specs'] = dic['name']
cleaned['name'] = prd
cleaned['data'] = [ (timestamp2datetime(int(price[0]) // 1000), price[1]) for price in dic['data'] ]
cleaned['access_time'] = datetime.utcnow().isoformat()
dics += json.dumps(cleaned, encoding='utf-8') + '\n'
else:
get_logger(batch_id, today_str, '/opt/service/log/').info('not match')
get_logger(batch_id, today_str, '/opt/service/log/').info('start posting prd page to cache')
return process._cache.post(url, dics)
def process(url, batch_id, parameter, manager, other_batch_process_time, *args,
**kwargs):
today_str = datetime.now().strftime('%Y%m%d')
get_logger(batch_id, today_str,
'/opt/service/log/').info('process {}'.format(url))
if not hasattr(process, '_downloader'):
headers = {
'Cookie':
'AJSTAT_ok_times=1; ant_stream_5762b612883d9=1470748235/1519574204; ASP.NET_SessionId=rpdjsrnmq3ybp0f4cnbdewm1; __utmt=1; bow_stream_5762b612883d9=13; __utma=240343830.1666180114.1470705813.1470719553.1470752966.3; __utmb=240343830.6.10.1470752966; __utmc=240343830; __utmz=240343830.1470705813.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)',
'Host': 'datacenter.cngrain.com',
}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_regs'):
setattr(
process, '_regs', {
'home':
re.compile('http://datacenter.cngrain.com/NewPrice.aspx'),
'market':
re.compile(
'http://datacenter.cngrain.com/PriceMainMark.aspx\?MarketId=(.*)'
)
})
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
if not hasattr(process, '_pattern'):
setattr(
process, '_pattern', {
'market':
'http://datacenter.cngrain.com/PriceMainMark.aspx',
'history':
'http://datacenter.cngrain.com/amline/PriceMarkXml.aspx{}'
})
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout = int(timeout)
gap = max(gap - other_batch_process_time, 0)
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
if label == 'home':
data = {
#'__EVENTTARGET':, #最后两位表示页数，其值不能超过11
'__EVENTARGUMENT': '',
'__ZIPSTATE':
'H4sIAAAAAAAEAOy9B2AcSZYlJi9tynt/SvVK1+B0oQiAYBMk2JBAEOzBiM3mkuwdaUcjKasqgcplVmVdZhZAzO2dvPfee++999577733ujudTif33/8/XGZkAWz2zkrayZ4hgKrIHz9+fB8/Iv7vX/M3/e1+7V/34bQtd3Z+t5Nq2ebL9mWZTfNvV+Usr3d/t+9+dfL7H9dtMS3z50XTHi+z8ropmt3f7fOffJq12W96Mft1dze/fbL8vM6K5dnyvLJvPf7k1/iNfs3f4Nf6NW982eu6//Jv+uv+2IPd+3s7Bwc7u7/p7Lf5tc7xz6/1a9O/vx79S81+7Nf8NX7T3+bX+QW/AV77vfLr5rf5NX7Bb/j7/65nbb44qdbL9tf6DWe/zW8jL/2avyka/jpv8nftr/vr/Rd/y9/6n//xf+Yv+A1fZJfFRdbmX9Xlr/s73D1bzvJ3L+vqslhO83HWrN79Hi/PZp/t7sxm9LoF8rv8Wr/ur/ef/81/83/2d/15v8uvPfzW3i699WuFb/0Xf/Nf+F/9sX/iprd279Nbv3anr7/hj/7P/4G/n9767Qffopd+nU5Xf/ef8Z//cX/mxq726K1ft/MW0YXfGuzqIb3064Uv/Vf/wN//n/8Nf/DGrh7QW79+Z1R/0h/9X/wFmxHcp7d+g85bf+6f+5//nX/+RrqDgj/WJ8Yfv7mve/RWErz1Y//13/sn/9f/wJ9Nc7bxxV2m4m/62/xav+DX/8miKSZlPgeUXx8f0Yu/8d3n1QXJxhm/osT7rbiTX4s6+R3/8z/pD/vP/9Q/+L/8K//4/+LP+4P+iz/jb/ov/pi//D/7e//O/+zv+oP+q7/yj/I4wXL67/Jr/Da/xu/ya/5av8vstzny2NpM3Q1j/BEf/4iPN/Lxb9jp7m+/6a17YKnfKHzrv/zT/qz/6g/5QzfRHvT4jTv0+OP+zP/s7/nrNr2EWf5Nwpf+6z/yj//P/54/YiN+GNVv2sHvr/gT//O/9+/Z1BVo+Jt18Pu7/0GRk8GXMF2/ee+l/+ov/wc3vfQpvfRbdLjwb/vb//O/8y/Y9BKY8LfsCyT3NDy/6Oq36gn/DW/tgeq/dYfqf/afftNbIOBvE771n/1df8x/8bf/nRvfAlv8th0M/7K/+r/42/7yjW9hXL9dh4R/+T/4X/2ZG9XTHmj423cw/Hv+5BuEaw9K43foUOPP/1NuGhdk5HfsqsI/+D//yzYrUPT1O4WC/J//EX/4f/Wn/Nn/+Z/4l2188YBe/J27gvx3ytAGeQov/YLOS3/G3/Rf/hkbpWsPb7Ht+11+nTlrRjFqv3XUqLE6u6HBbwhL9xvC0v0Gaul+q1/wYy+zi1y8uf1f8Fv//r8rXL3X1bqe5s7N++N+g5k4h1tsqn6r0Db+rkS4//rv/cP+8z/hH/zP/4g/7z/7e/5K6vq/+tv++v/6T/uD/vM/6Y/6z/7+P++//Mv+ARrlL7z7si6m+RfkyH6R1W9llPgtb2mo/8Wf/1f9F3/2n0pqdAYTKqP4bQnmf/kn/NH/5d/8t6bBM4O9lCbE9n/0f/n3/JUz2EL55NfffbCzM94BTzhL1/ni19WPfuy/+NP+DkJUSPPr6Ye/zt4OG1dD7B+zg5vByihq9sMeamYOEoLz6fbOwfbOQ0bvh0W13/y//LP//v/8b/oT/+u/56/Y+i/+7L/hv/6D/pw7Ab3+nj+uQ6+9nf04vfZ2HtyWXtAR3yy9fp0+vX478cD+8z/8jya9SpD+y7/lb/iv/9K/8D//m/8gIlO6kUzl/Kc9Av3Yf/G3/N3/1Z/45/2Xf/Pfv5mR9vcGGMl98V6E+Y18/D3aBOO6FXF+3Z894vz2wjr/5d/yR4cTFUjdp//lH/13/+d/1N9BXjSRrNtu+Am47uGnA1xnv3DEpd7+xD/5h0XcX+9nj7i/w3/xZ/85/+Xf/Pf9l3/V3/tf/U1/Z482G1hx79MBVtz7dO+2rOjrtG+OWr/+zx61flul1gb1v0+M+F/+9T1W3fz4hL330NCvQ1j3xc8RG/4GP4uEFe1HarBHGEtYip/eS7bx+IT99OEAx3560FeeP1TC/tjPHmFvML19sd45GDK95osbxfpnh0hJn0i/haQy/vN/4O+k8Om//Cv+7v/8L/wTiDy/+0byXGXLi+tieeHR6Nf7L/7OP4isi0eY30hoZT935NkdEE73hSXPf/5H/j3/+Z/wh2wmz68nuHuE+W3kk5CNN5DlN/xZI8utfN2e77Z7MGBF3Rc3MpBvFz6cQr9Rn0K/QLjvv/4j/+T/8m/5U0g2/rO/68/5L/68P/q//rP/pP/8D//r/vM/8e8iav34Rmpd/zTly6dxMfvP/r4/67/62/4qX8z2/+s/9E+mb+59+mkX5w1PwHX7Q7bWfnEjTWNC+ff+HRRpR4SSP+8gE6ftb/xzTdvkP/+7/qj/4i/6B3b2tpH28Sg2FEHYL36OKPab/FxT7Deg4Ju+4STK/wfo9Zv+cOkVNZJ/UNdI3huKFOwXP0fU+s361Ppd//O/7y/6z/+Gv+M//xP/0v/8j/iDKOdKngS5WeLg/Od/1x/yn/95yIt+tJFgP/jpSbny6PV1LcMAj0UsA2zn3/On3pQFMQPzSPbb2g97qMUp9pv/cCgGe0yQNgUO/8Vf9EcSq4Xf3/T49L2/M2Al3Bc3+reB5RWcP4S4v8UPhbi/+X/+l/2V/9Xf/EdswUH+U/9g0gY3pJf2H94fyKLYL37YhPot+4T6LWSV6b/88/7g//pP/wf/iz/rLyfiEGV+t82UKd/Nd/fuebT5rf+rP+Zv/i//tL/wP/t7/+AOHsPq7TfY3R/gJP8bS6H/+s/9O/7zP+1vuMmJ47F4FPptZFAheQYIdDCTZYyfFQLdRjIR0n9Q5Dkkme6LHzI5f+ufNXL+l3/PX0KmlMSxh8n/xzgOru2vQ4jITD08eDD69OEBcOYPfu39+ztmmZHXvM/tP78Wr93/5r82VjR+wW94XJbVFS1nUGw1/wW/AdY1Xhc/yH+tX/MX/IayrPGEFjNmF7Tq8RvMfs3f7Lf5NfDfb/jb/Dq/4Dc4aZqTMmuaX/c3yuq2mJb5718WTfsLfp3f/3d9/eTX+rWkK5pBYMMLMVjk//XcjDIN9ncf3jvY0S/dTH8Cn+nv/Tv/iz/9b/vP/v4/9b/44/7g//Kv+ltTuBl/3J/5X/xdf6PYdfr6v/o7/tj//O/7m4kF7sta0LGg8ZNFfiVc8AYrT7+Q/pFufuExL3ztPKT/7u3AG/+1gl5B3Xvbu3vbe/cE419rCOPdPsb/5T/wJxBL/dd/3h/5X/3xf/F/9Tf9oYR6+p//uX8/ZfCROP/D/pD/6m//m/7zP+Yf/C//jL/lZozvWYx33xvjX3sA4wcHm2n8X/69fxtorJqGCUwy8p//3X/4f/HH/HX/+R//R96Wxg8Obo3x7qeC8a8zhPHDHsZjIigt+RCZdcnnL//7/6u//69LhcBimckR/S//3r9eZuG2ZH7w8L2R/nWHkP60T+aApn/uX/Bf/Dl/Rkq5bPwhGAuZ/9a/5D//Y/6qW5P501tjvKOM8esNYfygjzGR+b/+s/9wIjOxB1bI/46/KmBlYpT/7O/66ymeuZkxHI0fvDfGv/4Qxvt9jIWV/+u/9s/6z/7+P54Y+r/8k/6IgJXB5n/R3/ef/X3QI7em8f4tMd7d3j0QjH+DIYzv9zAek7qg9eX//E/+Y/7zP+qv/s//sj9H/MOAzL4quTWZ77830mJa+Fe4eKS1jWn4dX6XH+tp/d8lCbX97/IbDmn6Xye03R/9F3/eX/ef/91/23/+9/wV5AP/l3/aX0VjJFH+r//IP3Hv/t7Bf/Z3/ZH/+Z8E7n88b9vVo7t3r6v1D+bFeLq8qMmsj6fV4u68XZR3dfTNXdhBSvc+vLu7v3//3qdjfBsQObSWD5UOHaQ+Pvgv/rw/anfvv/gz/3IQ/m+gtbm/7b/4M/+O//yv/7P+8z/kr5IZ+S//2r9gPB5/bcz2Ht4Gs1+7j9nWA8Lsv/ij/mTgxYj8p3/QH2wJ91/9g38ugpM/8Y/7z/+Sv+Dro3b/Nqj9On3U7vznf9zf85//hX/hf/UX/3H/xR/6h/+Pf9+fAxo+BAn/7n/wv/rL/8H//M/7m+h7cuk/hGwPdm+D26/7c4Pbzm1w+/XeCzfoqm8Ct09vxW6//s8Nbge3we03eC/cvqk5/fTBbXDzNOVvLb/9Nps861/vd/n157/Lb/Br/Zp44wYNOvt/AgAA//+cf8MQDy8AAA==',
'__VIEWSTATE': '',
#'__EVENTVALIDATION':,
'ctl00$txtTitle': '',
}
content = process._downloader.downloader_wrapper(
url,
batch_id,
gap,
method='get',
timeout=timeout,
refresh=True,
encoding='utf-8'
) #第一页使用get,得到ZIPSTATE参数,之后在循环内持续利用__EVENTTARGET参数翻页同时更新ZIPSTATE参数
if content == '':
return False
while 1:
dom = lxml.html.fromstring(content) #开始解析当前页
market_suffixes = dom.xpath(
'//a[contains(@href,"MarketId")]/@href')
if not market_suffixes:
get_logger(batch_id, today_str,
'/opt/service/log/').info('No market_suffixes')
get_logger(batch_id, today_str,
'/opt/service/log/').info(content)
return False
market_suffixes_set = set(
market_suffixes) #去重，但是对于市场名重复延续到下一页的情况无效，会重复多爬一次
market_url_list = [
urlparse.urljoin(process._pattern['market'], suffix)
for suffix in market_suffixes_set
]
manager.put_urls_enqueue(
batch_id, market_url_list) #完成本页的处理，将市场名入队，接下去的操作全是为了翻页
page_label = dom.xpath('//td[@colspan="10"]//span')[
0] #在所有页数里，只有当前页的标签是span，定位到当前页
page = page_label.xpath('.//text()')
if page == ['40']:
return True
next_sibling_list = (
page_label.xpath('./following-sibling::a')
) #定位下一页，下一页不存在时则结束，（即使是网页上的...按钮，在此种判断里也会算存在下一页）
if not next_sibling_list:
return True
next_sibling = next_sibling_list[0]
next_raw_js = next_sibling.xpath(
'./@href'
)[0] # 其形式为 : "javascript:__doPostBack('ctl00$ContentPlaceHolder1$DG_FullLatestPrice$ctl24$ctl01','')"
eventtarget = re.findall('\(\'(.*)\',', next_raw_js)[0]
data['__EVENTTARGET'] = eventtarget
last = data[
'__ZIPSTATE'] #用来储存上一次的ZIPSTATE参数，如果新参数失败就换旧的使用——实践过程中发现某页的ZIPSTATE有小概率对下一页失效
data['__ZIPSTATE'] = (
dom.xpath('//input[@name="__ZIPSTATE"]/@value'))[0]
data['__EVENTVALIDATION'] = (dom.xpath(
'//input[@name="__EVENTVALIDATION"]/@value'))[0] #更新参数
for _ in range(0, 3): #开始对下一页发请求，绝大多数失败都发生在这一步，慎重
try:
content = process._downloader.downloader_wrapper(
url,
batch_id,
gap,
method='post',
timeout=timeout,
refresh=True,
data=data,
encoding='utf-8')
if content == '' or 'sent a request that this server could not understand' in content or 'bad request' in content:
get_logger(
batch_id, today_str,
'/opt/service/log/').info('change ZIPSTATE')
get_logger(
batch_id, today_str,
'/opt/service/log/').info('change ZIPSTATE')
data[
'__ZIPSTATE'] = last #使用上一次的参数，不考虑连续两次失败，实际调试中也没遇到过
continue
except Exception, e:
get_logger(batch_id, today_str,
'/opt/service/log/').info(e)
continue
break
else:
get_logger(batch_id, today_str,
'/opt/service/log/').info('failed 3 times')
return False
elif label == 'market': #开始处理市场页，同时在此处理价格历史，加入到产品信息生成结果
get_logger(batch_id, today_str,
'/opt/service/log/').info('in market page')
market_id = url[url.find('=') + 1:]
url = url.replace(market_id, urllib.quote(market_id))
content = process._downloader.downloader_wrapper(
url,
batch_id,
gap,
timeout=timeout,
refresh=True,
encoding='utf-8',
redirect_check=False)
dom = lxml.html.fromstring(content)
title = dom.xpath('//a[@class="this_tab"]//text()')
if title:
title = title[0]
result = {}
result['market'] = title.strip()
result['product_list'] = []
table_node = dom.xpath('//table[@class="data_table"]')[0] #得到产品表格
products_nodes = table_node.xpath('.//tr')[1:-1] #去掉表头和尾巴
newest_time = None
for product_node in products_nodes: #市场页会有相同产品在不同时间的批次，以此为根据去重
report_time = product_node.xpath('./td[9]/text()')
if not newest_time:
newest_time = report_time
if newest_time != report_time:
break
relative_path = product_node.xpath('./td[10]/a/@href')[0]
history_url = process._pattern['history'].format(relative_path)
get_logger(batch_id, today_str, '/opt/service/log/').info(
'The history_url is :{}'.format(history_url))
content = process._downloader.downloader_wrapper(
history_url,
batch_id,
gap,
timeout=timeout,
refresh=True,
encoding='utf-8')
if content: #有的价格历史会显示‘数据还在完善中‘
dom_history = lxml.html.fromstring(content)
date_list = dom_history.xpath('//series//value/text()')
price_list = dom_history.xpath('//graph//value/text()')
history_dic = dict(zip(date_list, price_list))
else:
history_dic = {}
product_item = {
'variety':
product_node.xpath('./td[1]/text()')[0].strip(),
'level':
product_node.xpath('./td[2]/text()')[0].strip(),
'price_type':
product_node.xpath('./td[5]/text()')[0].strip(),
'produce_year':
product_node.xpath('./td[6]/text()')[0].strip(),
'produce_area':
product_node.xpath('./td[7]/text()')[0].strip(),
'deliver_area':
product_node.xpath('./td[8]/text()')[0].strip(),
'source':
'http://datacenter.cngrain.com{}'.format(relative_path),
'access_time':
datetime.utcnow().isoformat(),
'price_history':
history_dic,
}
result['product_list'].append(product_item)
result['market_source'] = url
# print (json.dumps(result, ensure_ascii = False))
return process._cache.post(url,
json.dumps(result, ensure_ascii=False))
def process(url, batch_id, parameter, manager, other_batch_process_time, *args,
**kwargs):
today_str = datetime.now().strftime('%Y%m%d')
get_logger(batch_id, today_str,
'/opt/service/log/').info('process {}'.format(url))
if not hasattr(process, '_downloader'):
headers = {}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_regs'):
setattr(process, '_regs', {
'first_letter': re.compile('^[A-Z]$'),
'drug': re.compile('(\d+)')
})
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
method, gap, js, timeout, data = parameter.split(':')
gap = float(gap)
timeout = int(timeout)
gap = max(gap - other_batch_process_time, 0)
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
if label == 'first_letter':
first_letter = url
data = {
'Data':
'{{\"url\": \"\", \"letter\": \"{}\"}}'.format(
first_letter) # form data示例： Data:{"letter":"J","url":""}
}
query_url = 'http://yaocai.zyctd.com/Ajax/AjaxHandle.ashx?CommandName=common/MCodexService/GetCodexNameByLetter'
content = process._downloader.downloader_wrapper(query_url,
batch_id,
gap,
method='post',
data=data,
timeout=timeout,
refresh=True)
if not content:
return False
drug_list = json.loads(content)
MBID_list = []
for drug in drug_list[u'Data']:
MBID_list.append(str(drug[u'MBID']))
manager.put_urls_enqueue(batch_id, MBID_list) # 每个MBID为一个数字
elif label == 'drug':
mbid = url
query_url = 'http://www.zyctd.com/Breeds/GetMCodexPoolListByMBID'
data = {'mbid': '{}'.format(mbid), 'IsMarket': 'true'}
content = process._downloader.downloader_wrapper(query_url,
batch_id,
gap,
method='post',
data=data,
timeout=timeout,
refresh=True)
if not content:
return False
item = json.loads(content)
sub_drug_list = item[u'Data']
if not sub_drug_list: # 请求成功然而列表为空，说明这种药材没有价格数据，属正常情况
return True
for sub_drug in sub_drug_list: # eg：一个刀豆list里根据不同规格和产地的刀豆会分为不同的sub_drug,拥有不同的MBSID
price_history_url = 'http://www.zyctd.com/Breeds/GetPriceTrend'
data = {'MBSID': sub_drug['MBSID'], 'IsMarket': 'true'}
price_content = process._downloader.downloader_wrapper(
price_history_url,
batch_id,
gap,
method='post',
data=data,
timeout=timeout,
refresh=True)
if not price_content:
return False
spec_info = sub_drug['MSpec'].split(' ')
productGrade = spec_info[0]
if len(spec_info) == 2:
productPlaceOfOrigin = spec_info[
1] # MSpec一般情况示例： 大统 东北 OR 统 较广
else: # MSpec特殊情况示例： 统
productPlaceOfOrigin = ''
price_item = json.loads(price_content)[u'Data']
price_data = price_item[
u'PriceChartData'] # 注意price_data是一个字符串，需要再次loads后变为一个列表，每个列表代表一个药市，其中还嵌套着价格列表,价格里时间表示为时间戳等。
if price_data == '[]': # 即使存在这种规格，也有可能会没有价格历史
return True
formatted_price_data = deal_with_price(price_data)
result_item = {
'name': sub_drug['MName'],
'productGrade': productGrade,
'productPlaceOfOrigin': productPlaceOfOrigin,
'source':
'http://www.zyctd.com/jiage/xq{}.html'.format(mbid),
'access_time': datetime.utcnow().isoformat(),
'price_data': formatted_price_data
}
if not process._cache.post(str(sub_drug['MBSID']),
json.dumps(result_item,
ensure_ascii=False),
refresh=True):
return False
return True
def work(self, batch_id, queue_dict, *args, **kwargs):
batch_key_filename = batch_id.rsplit('-', 1)[0].replace('-', '_')
try:
module = __import__('workers.{}'.format(batch_key_filename),
fromlist=['process'])
except:
module = __import__('workers.prefetch', fromlist=['process'])
while 1:
today_str = datetime.now().strftime('%Y%m%d')
if kwargs and kwargs.get("debug"):
get_logger(
batch_id, today_str,
'/opt/service/log/').info('begin get items from queue')
results = queue_dict['queue'].get(block=True,
timeout=3,
interval=1)
if kwargs and kwargs.get("debug"):
get_logger(
batch_id, today_str,
'/opt/service/log/').info('finish get items from queue')
if not results:
break
for url_id in results:
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str, '/opt/service/log/').info(
'begin get url from thinhash redis')
# TODO change to hmget
url = queue_dict['thinhash'].hget(url_id)
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str, '/opt/service/log/').info(
'end get url from thinhash redis')
try:
process_status = module.process(
url, batch_id, self._batch_param[batch_id],
self.manager, *args, **kwargs)
except Exception as e:
Record.instance().add_exception(batch_id, url, repr(e))
queue_dict['queue'].task_done(url_id)
continue
if process_status:
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str,
'/opt/service/log/').info(
'begin task done for record redis')
queue_dict['queue'].task_done(url_id)
Record.instance().increase_success(batch_id)
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str,
'/opt/service/log/').info(
'end task done for record redis')
def process(url, batch_id, parameter, manager, other_batch_process_time, *args,
**kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
if not hasattr(process, '_regs'):
setattr(
process, '_regs', {
'main':
re.compile(
r'http://jiage.cngold.org/jinshubi/list_3640_(\d+).html'),
'info':
re.compile(
r'http://jiage.cngold.org/c/(\d+-\d+-\d+)/c(\d+).html'),
'index':
re.compile(r'http://jiage.cngold.org/jinshubi/index.html')
})
method, gap, js, timeout, data = parameter.split(':')
gap = float(max(0, float(gap) - other_batch_process_time))
timeout = int(timeout)
today_str = datetime.now().strftime('%Y%m%d')
if url == 'http://jiage.cngold.org/jinshubi/list_3640_1.html':
url = 'http://jiage.cngold.org/jinshubi/index.html'
# if kwargs and kwargs.get("debug"):
# get_logger(batch_id, today_str, '/opt/service/log/').info('start download')
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout)
# print(content)
if content == '':
get_logger(batch_id, today_str,
'/opt/service/log/').info(url + ' no content')
return False
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
# print("not match")
continue
page = etree.HTML(content)
if label == 'index':
prices = page.xpath(".//ul[@class='list_baojia']/li/a/@href")
# get_logger(batch_id, today_str, '/opt/service/log/').info(str(prices))
manager.put_urls_enqueue(batch_id, prices[:3])
return True
elif label == 'info':
dic = {}
datestr = m.group(1)
table = page.xpath(".//table//td/text()")
table = [t.strip() for t in table]
dic[u'产品名称'] = table[0]
dic[u'产品价格'] = table[1]
dic[u'价格单位'] = table[2]
dic[u'涨跌'] = table[3]
dic[u'日期'] = datestr
dic[u'source'] = url
dic[u'access_time'] = datetime.utcnow().isoformat()
data = json.dumps(dic, ensure_ascii=False)
# get_logger(batch_id, today_str, '/opt/service/log/').info(data)
return process._cache.post(url, data)
def process(url, batch_id, parameter, manager, other_batch_process_time, *args,
**kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
if not hasattr(process, '_regs'):
setattr(
process, '_regs', {
'main':
re.compile(
r'http://www.zysj.com.cn/zhongyaocai/index__\d+.html'),
'prd':
re.compile(
r'http://www.zysj.com.cn/zhongyaocai/yaocai_\w/(.+?).html')
})
method, gap, js, timeout, data = parameter.split(':')
gap = float(max(0, float(gap) - other_batch_process_time))
timeout = int(timeout)
today_str = datetime.now().strftime('%Y%m%d')
# if kwargs and kwargs.get("debug"):
# get_logger(batch_id, today_str, '/opt/service/log/').info('start download')
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout)
# print(content)
if content == '':
# print("no content")
get_logger(batch_id, today_str,
'/opt/service/log/').info(url + ' no content')
return False
# content.encoding='gb18030'
# if kwargs and kwargs.get("debug"):
# get_logger(batch_id, today_str, '/opt/service/log/').info('start parsing url')
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
page = etree.HTML(content)
if label == 'main':
get_logger(batch_id, today_str,
'/opt/service/log/').info("adding Chinese Meds")
meds = page.xpath("//*[@id=\"list\"]/ul/li/a/@href"
) # links for meds in main page
meds = [urlparse.urljoin(SITE, med) for med in meds]
# print(meds[:5])
get_logger(batch_id, today_str,
'/opt/service/log/').info('adding Meds urls into queue')
manager.put_urls_enqueue(batch_id, meds)
return True
elif label == 'prd':
med_name = page.xpath("//*[@id=\"article\"]/h1/text()")[0]
get_logger(batch_id, today_str,
'/opt/service/log/').info(med_name + " main page")
# print(med_name,"main page")
book_list = []
dictionary = {}
books = content.split('<hr />') # 用来分开不同的药典
if len(books) == 2: # 只有一个药典的情况
books = [books[0]]
else: # 有多个药典的情况
books = books[1:-1]
for book in books:
page = etree.HTML(
book.replace('<strong>',
'').replace('</strong>', '').replace(
'<sub>', '').replace('</sub>', ''))
med_info = page.xpath("//p/text()")
data = {}
# data['source'] = url
dictionary['source'] = url
# data['access_time'] = datetime.utcnow().isoformat()
dictionary['access_time'] = datetime.utcnow().isoformat()
data_list = []
for info in med_info:
m = re.compile(r'【.+?】').match(info.encode('utf-8'))
if m:
prop = m.group(0)[3:-3]
cleaned = re.sub(r'【.+?】', '', info.encode('utf-8'))
data[prop] = cleaned
data_list.append({prop: cleaned})
else:
data[prop] += '\n' + info.encode('utf-8')
data_list[-1][prop] += '\n' + info.encode('utf-8')
book_name = data['摘录']
# dics[book_name] = data
book_list.append({book_name: data_list}) # 为了保持原书籍的顺序，使用列表结构
dictionary[data['药材名称']] = book_list
dictionary = json.dumps(dictionary,
encoding='utf-8',
ensure_ascii=False)
get_logger(
batch_id, today_str,
'/opt/service/log/').info('start posting prd page to cache')
return process._cache.post(url, dictionary)
def process(url, batch_id, parameter, manager, other_batch_process_time, *args,
**kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
if not hasattr(process, '_regs'):
setattr(
process, '_regs', {
'main':
re.compile(
r'http://chem.100ppi.com/price/plist-(\d+)(-{1,3})(\d+).html'
),
'prd':
re.compile(r'http://www.100ppi.com/price/detail-(\d+).html')
})
def safe_state(statement):
return statement[0] if statement else ''
method, gap, js, timeout, data = parameter.split(':')
gap = float(max(0, float(gap) - other_batch_process_time))
timeout = int(timeout)
today_str = datetime.now().strftime('%Y%m%d')
# print(url)
# if kwargs and kwargs.get("debug"):
# get_logger(batch_id, today_str, '/opt/service/log/').info('start download')
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout)
# print(content)
if content == '':
get_logger(batch_id, today_str,
'/opt/service/log/').info(url + ' no content')
return False
# content.encoding='gb18030'
# if kwargs and kwargs.get("debug"):
# get_logger(batch_id, today_str, '/opt/service/log/').info('start parsing url')
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
page = etree.HTML(content)
if label == 'main':
# print("adding chem prds")
prd_links = page.xpath('//table/tr/td[1]/div/a/@href')
if not prd_links:
# print('end of pages')
get_logger(batch_id, today_str,
'/opt/service/log/').info('end of pages')
return True
next_pat = re.compile(r'plist-(\d+)(-{1,3})(\d+).html')
current = next_pat.search(url)
current = str(int(current.group(3)) + 1)
next_page = url[:url.rfind('-') + 1] + current + '.html'
prd_links.append(urlparse.urljoin(SITE, next_page))
get_logger(batch_id, today_str,
'/opt/service/log/').info('||'.join(prd_links) +
' added to queue')
manager.put_urls_enqueue(batch_id, prd_links)
return True
else:
data = {}
data['name'] = page.xpath(
"/html/body/div[8]/div[1]/span[2]/text()")[0]
# print(data['name'], 'prd page')
data['source'] = url
# data['prd_header'] = page.xpath("//div[@class=\"mb20\"]/table/tr/th/text()")
# data['prd_infos'] = page.xpath("//div[@class=\"mb20\"]/table/tr/td/text()")
prd_header = page.xpath(
"/html/body/div[8]/div[2]/div[1]/div[1]/h3/text()")[0]
idx_left, idx_right = prd_header.find(u'('), prd_header.find(u')')
data[u'报价类型'] = prd_header[idx_left + 1:idx_right]
data[u'报价机构'] = page.xpath(
"/html/body/div[8]/div[2]/div[2]/div[2]/table/tr[1]/td/h3/text()"
)[0].strip()
data[u'商品报价'] = safe_state(
page.xpath("//div[@class=\"mb20\"]/table/tr[1]/td[1]/text()"))
data[u'发布时间'] = safe_state(
page.xpath("//div[@class=\"mb20\"]/table/tr[1]/td[2]/text()"))
data[u'出产地'] = safe_state(
page.xpath("//div[@class=\"mb20\"]/table/tr[2]/td[1]/text()"))
data[u'有效期'] = safe_state(
page.xpath("//div[@class=\"mb20\"]/table/tr[2]/td[2]/text()"))
data[u'仓储地'] = safe_state(
page.xpath("//div[@class=\"mb20\"]/table/tr[3]/td[1]/text()"))
data[u'包装说明'] = safe_state(
page.xpath("//div[@class=\"mb20\"]/table/tr[3]/td[2]/text()"))
data[u'生产厂家'] = safe_state(
page.xpath(
"/html/body/div[8]/div[2]/div[1]/div[2]/div/div[2]/text()")
)
info = {}
table_header = page.xpath(
"//table[@class=\"mb20 st2-table tac\"]/tr/th/text()")
table_content = page.xpath(
"//table[@class=\"mb20 st2-table tac\"]/tr/td/text()")
for header, cont in zip(table_header, table_content):
info[header] = cont
data[u'详细信息'] = info
contact = {}
contact[u'联系人'] = safe_state(
page.xpath(
"//div[@class=\"connect\"]/table/tr[2]/td[2]/text()"))
contact[u'电话'] = safe_state(
page.xpath(
"//div[@class=\"connect\"]/table/tr[3]/td[2]/text()"))
contact[u'传真'] = safe_state(
page.xpath(
"//div[@class=\"connect\"]/table/tr[4]/td[2]/text()"))
contact[u'邮件'] = safe_state(
page.xpath(
"//div[@class=\"connect\"]/table/tr[5]/td[2]/text()"))
contact[u'手机'] = safe_state(
page.xpath(
"//div[@class=\"connect\"]/table/tr[6]/td[2]/text()"))
contact[u'地址'] = safe_state(
page.xpath(
"//div[@class=\"connect\"]/table/tr[7]/td[2]/text()"))
contact[u'网址'] = safe_state(
page.xpath(
"//div[@class=\"connect\"]/table/tr[8]/td[2]/text()"))
data[u'联系方式'] = contact
# print(json.dumps(data, encoding='utf-8', ensure_ascii=False))
dics = json.dumps(data, encoding='utf-8', ensure_ascii=False)
get_logger(batch_id, today_str,
'/opt/service/log/').info(dics + ' saved to S3')
return process._cache.post(url, dics)
def process(url, batch_id, parameter, manager, *args, **kwargs):
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader',
DownloadWrapper('s3', headers, REGION_NAME))
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CacheS3(head + '-json-' + tail))
if not hasattr(process, '_regs'):
setattr(
process, '_regs', {
'entity':
re.compile(
urlparse.urljoin(SITE,
'cndbpedia/api/entity\?mention=(.+)')),
'avp':
re.compile(
urlparse.urljoin(SITE,
'cndbpedia/api/entityAVP\?entity=(.+)')),
'info':
re.compile(
urlparse.urljoin(
SITE, 'cndbpedia/api/entityInformation\?entity=(.+)')),
'tags':
re.compile(
urlparse.urljoin(SITE,
'cndbpedia/api/entityTag\?entity=(.+)')),
})
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout = int(timeout)
today_str = datetime.now().strftime('%Y%m%d')
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str,
'/opt/service/log/').info('start download')
content = process._downloader.downloader_wrapper(url,
batch_id,
gap,
timeout=timeout,
encoding='utf-8')
if content == '':
return False
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str,
'/opt/service/log/').info('start parsing url')
for label, reg in process._regs.iteritems():
m = reg.match(url)
if not m:
continue
entity = urllib.unquote(m.group(1))
if label == 'entity':
urls = []
avpair_api = urlparse.urljoin(SITE,
'cndbpedia/api/entityAVP?entity={}')
info_api = urlparse.urljoin(
SITE, 'cndbpedia/api/entityInformation?entity={}')
tags_api = urlparse.urljoin(SITE,
'cndbpedia/api/entityTag?entity={}')
js = json.loads(content)
for ent in js[u'entity']:
if isinstance(ent, unicode):
ent = ent.encode('utf-8')
ent = urllib.quote(ent)
urls.append(avpair_api.format(ent))
urls.append(info_api.format(ent))
urls.append(tags_api.format(ent))
manager.put_urls_enqueue(batch_id, urls)
return True
else:
data = json.dumps({entity: json.loads(content)})
if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str, '/opt/service/log/').info(
'start post {} json'.format(label))
return process._cache.post(url, data)
def process(url, batch_id, parameter, manager, other_batch_process_time, *args, **kwargs):
today_str = datetime.now().strftime('%Y%m%d')
get_logger(batch_id, today_str, '/opt/service/log/').info(url)
home_page = 'http://app1.sfda.gov.cn/datasearch/face3/base.jsp?tableId=36&tableName=TABLE36&title=%BD%F8%BF%DA%D2%A9%C6%B7&bcId=124356651564146415214424405468'
if not hasattr(process, '_downloader'):
domain_name = Downloader.url2domain(url)
headers = {'Host': domain_name}
setattr(process, '_downloader', DownloadWrapper(None, headers))
if not hasattr(process,'_reg'):
setattr(process, '_reg', {
'detail': re.compile('http://app1.sfda.gov.cn/datasearch/face3/content.jsp\?tableId=36&tableName=TABLE36&Id=(\d+)'),
})
if not hasattr(process, '_cache'):
head, tail = batch_id.split('-')
setattr(process, '_cache', CachePeriod(batch_id, CACHE_SERVER))
method, gap, js, timeout, data = parameter.split(':')
gap = int(gap)
timeout= int(timeout)
gap = max(gap - other_batch_process_time, 0)
#if kwargs and kwargs.get("debug"):
get_logger(batch_id, today_str, '/opt/service/log/').info('start download')
data = {
'tableId' : '36',
'State' : '1',
'bcId' : '124356651564146415214424405468',
'State' : '1',
'State' : '1',
'tableName' : 'TABLE36',
'State' : '1',
'viewtitleName' : 'COLUMN361',
'State' : '1',
'viewsubTitleName' : 'COLUMN354,COLUMN355,COLUMN356,COLUMN823',
'curstart':'2',
'State' : '1',
'State' : '1',
}
if url == home_page:
#if kwargs and kwargs.get("debug"):
page = 1
while 1 :
time.sleep(gap)
get_logger(batch_id, today_str, '/opt/service/log/').info('start parsing url at page {}'.format(page))
data['curstart'] = page
content = process._downloader.downloader_wrapper('http://app1.sfda.gov.cn/datasearch/face3/search.jsp',
batch_id,
gap,
method = 'post',
timeout = timeout,
refresh = True,
data = data,
encoding = 'utf-8'
)
#get_logger(batch_id, today_str, '/opt/service/log/').info(content)
ids = re.findall(u'进口药品&Id=(\d+)', content)
get_logger(batch_id, today_str, '/opt/service/log/').info(ids)
if not ids:
get_logger(batch_id, today_str, '/opt/service/log/').info('End at {} pages'.format(page))
break
# if page == 3:
# break
get_logger(batch_id, today_str, '/opt/service/log/').info('ids : {}'.format(ids))
url_pattern = 'http://app1.sfda.gov.cn/datasearch/face3/content.jsp?tableId=36&tableName=TABLE36&Id={}'
urls = []
for drug_id in ids:
url = url_pattern.format(drug_id)
urls.append(url)
manager.put_urls_enqueue(batch_id, urls)
page += 1
get_logger(batch_id, today_str, '/opt/service/log/').info('going to page{}'.format(page))
return True
elif process._reg['detail'].match(url):
content = process._downloader.downloader_wrapper(
url,
batch_id,
gap,
timeout = timeout,
refresh = True
)
if content == '':
return False
dom = lxml.html.fromstring(content)
table = dom.xpath('//tr')
item = {
'license_number': table[1].xpath('./td')[1].xpath('./text()'), # [u'注册证号']
'old_license_number': table[2].xpath('./td')[1].xpath('./text()'), # [u'原注册证号']
'packaging_license_number': table[4].xpath('./td')[1].xpath('./text()'), # [u'分包装批准文号']
'company_chs': table[5].xpath('./td')[1].xpath('./text()'), # [u'公司名称（中文）']
'company_eng': table[6].xpath('./td')[1].xpath('./text()'), # [u'公司名称（英文）']
'product_name_chs': table[11].xpath('./td')[1].xpath('./text()'), # [u'产品名称（中文）']
'product_name_eng': table[12].xpath('./td')[1].xpath('./text()'), # [u'产品名称（英文）']
'commodity_name_chs': table[13].xpath('./td')[1].xpath('./text()'), # [u'商品名（中文）']
'commodity_name_eng': table[14].xpath('./td')[1].xpath('./text()'), # [u'商品名（英文）']
'drug_form': table[15].xpath('./td')[1].xpath('./text()'), # [u'剂型（中文）']
'specification': table[16].xpath('./td')[1].xpath('./text()'), # [u'规格（中文）']
'dosage': table[17].xpath('./td')[1].xpath('./text()'), # [u'包装规格（中文）']
'manufacturer_chs': table[18].xpath('./td')[1].xpath('./text()'), # [u'生产厂商（中文）']
'manufacturer_eng': table[19].xpath('./td')[1].xpath('./text()'), # [u'生产厂商（英文）']
'manuf_address_chs': table[20].xpath('./td')[1].xpath('./text()'), # [u'厂商地址（中文）']
'manuf_address_eng': table[21].xpath('./td')[1].xpath('./text()'), # [u'厂商地址（英文）']
'manuf_country_chs': table[22].xpath('./td')[1].xpath('./text()'), # [u'厂商国家/地区（中文）']
'manuf_country_eng': table[23].xpath('./td')[1].xpath('./text()'), # [u'厂商国家/地区（英文）']
'packaging_company_name': table[26].xpath('./td')[1].xpath('./text()'), # [u'分包装企业名称']
'packaging_company_address': table[27].xpath('./td')[1].xpath('./text()'), # [u'分包装企业地址']
'category': table[31].xpath('./td')[1].xpath('./text()'), # [u'产品类别']
'standard_code': table[32].xpath('./td')[1].xpath('./text()'), # [u'药品本位码']
'source' : [url], #设为list格式与之前字段统一，在下面的循环里一并取出
}
for k,v in item.iteritems():
if len(v) > 0:
item[k] = v[0]
else :
item[k] = None
return process._cache.post(url, json.dumps(item, ensure_ascii = False))
