def task_validator(project, sstr, endwith, endtime): # 是否提前结束 if stats.c_skipped_pmid >= config.pmid_max_c_skip and endwith: # 设定提前完成并达到提前完成的skip数量 msg.msg("crawl pmid", project + sstr, "repeat end", "succ", "notice", msg.display, msg.log) return False # false是不通过 if endtime < ut.time_str("full") and endwith: # 设定提前完成并达到提前完成的时间上限 msg.msg("crawl pmid", project + sstr, "time end", "succ", "notice", msg.display, msg.log) return False else: return True # True是通过
def journal_name_wash(journal_name_raw): # 原始名称清洗(主要针对各种括号和标点、解释、注释) re_bracket = re.compile("[\\[\\(](.*?)[\\]\\)]") # 去处括号解释 re_explaination = re.compile(" ??[:=].*") # 去处冒号后的解释 journal_name = journal_name_raw.replace('&', "&").replace(',', '').replace( ".", '') # &是部分名称中包含的 journal_name = re_bracket.sub('', journal_name) journal_name = re_explaination.sub('', journal_name) journal_name = journal_name.upper() # 清洗过的名称全大写 msg.msg("journal name", journal_name_raw, "washed", journal_name, "debug", msg.display) return journal_name
def parse_url(project, sstr="key_words"): # 把keyword变成链接形式,临时这样,未来增加内容 sstr_type = mh.read_sstr_type(project, sstr) if sstr_type == "key_words" or sstr_type == "key_word": if "," in sstr: sstr = sstr.replace(", ", ",").replace(" ,", ",") # 防止有空格 elif " " in sstr: sstr = sstr.replace(" ", ",") sstr = sstr.replace(",", "%2C") # 换成链接形式 if sstr_type == "expression": pass url = "https://www.ncbi.nlm.nih.gov/pubmed/?term=" + sstr # 最初的查询网址 msg.msg("url", sstr, "parsed", url, "debug", msg.display) return url
def journal_detail(journal_name): record = mh.read_journal_detail(journal_name) # 直接试一下 if record: msg.msg("journal record", journal_name, "local retrieved", "succ", "debug", msg.display) return record else: wjournal_name = journal_name_wash(journal_name) # 清洗过的在正式名里试一下 record = mh.read_ojournal_detail(wjournal_name) if record: msg.msg("journal record", journal_name, "local retrieved", "succ", "debug", msg.display) return record else: ojournal_name = get_official_name(wjournal_name) # 网络正式名在正式名里试一下 record = mh.read_ojournal_detail(ojournal_name) if record: msg.msg("journal record", journal_name, "web retrieved", "succ", "debug", msg.display) return record else: journal_info = get_journal_info(ojournal_name) # 网络正式名在网络查一下 journal_if = journal_info[0] journal_zone = journal_info[1] mh.add_journal(journal_name, ojournal_name, journal_if, journal_zone) # 新杂志储存 msg.msg("journal record", journal_name, "web retrieved", "succ", "debug", msg.display) data = journal_name, ojournal_name, journal_if, journal_zone return data
def run_task(project, sstr): # 多少时间后开始运行 record_number, mrmins, endwith = get_task_config(project, sstr) endtime = ut.time_str("full", mrmins) msg.msg("crawl pmid", project + sstr, "started", "succ", "important", msg.display, msg.log, msg.stat) pc.run_pmid_crawler(project, sstr, record_number, endwith, endtime) msg.msg("crawl pmid", project + sstr, "finished", "succ", "important", msg.display, msg.log, msg.stat) msg.msg("crawl detail", project + sstr, "started", "succ", "important", msg.display, msg.log, msg.stat) dc.run_detail_crawler(project, sstr, record_number) msg.msg("crawl detail", project + sstr, "finished", "succ", "important", msg.display, msg.log, msg.stat)
def get_data(filename, drop_data=False): '''Get pandas.DataFrame from .csv or .csv.zip''' msg('Reading datafile : {}'.format(filename)) if not os.path.exists(filename): raise IOError('No filename: {}'.format(filename)) if filename.endswith('.zip'): z = zipfile.ZipFile(filename) filename = z.open(filename.replace('.zip', '')) data = pandas.read_csv( filename, parse_dates=['Dates'], infer_datetime_format=True, comment='#', ) #data.Time = data.Dates.map(lambda x: x.time()) if drop_data: data = data[(data.X < -121) & (data.Y < 40)] data = data.dropna() data = data.reset_index(drop=True) msg('Read datafile : {}'.format(filename), 2) return data
def get_data(filename, drop_data=False): '''Get pandas.DataFrame from .csv or .csv.zip''' msg('Reading datafile : {}'.format(filename)) if not os.path.exists(filename): raise IOError('No filename: {}'.format(filename)) if filename.endswith('.zip'): z = zipfile.ZipFile(filename) filename = z.open(filename.replace('.zip', '')) data = pandas.read_csv( filename, parse_dates=['Dates'], infer_datetime_format=True, comment='#', ) #data.Time = data.Dates.map(lambda x: x.time()) if drop_data: data = data[(data.X < -121) & (data.Y < 40)] data = data.dropna() data = data.reset_index(drop=True) msg('Read datafile : {}'.format(filename), 2) return data
def draw(self): if self.perfect: msg(self.scr, self.message, wiggle(self.pos, 5 * self.size), self.color, self.fontsize * wiggle(self.size, 1, -2) * 1.2, centered=True, **self.kwargs) msg(self.scr, self.message, wiggle(self.pos, 5 * self.size), self.color, self.fontsize * wiggle(self.size, 1, -2) * 1.2, centered=True, **self.kwargs) else: msg(self.scr, self.message, self.pos, self.color, self.fontsize * self.size, centered=True, **self.kwargs)
def get_official_name(journal_name_raw, proxy=None): # 查找杂志的全名,支持模糊查询,只输出最符合的那个 url = "http://www.letpub.com.cn/journalappAjax.php?querytype=autojournal&term=" + \ journal_name_raw.replace("&", "%26").replace(" ", "+") tries = config.request_dp_tries while tries > 0: try: opener = requests.Session() doc = opener.get(url, timeout=20, headers=agents.get_header()).text list = doc.split('},{') # 获取列表,但是只有最match的被采纳 journal_name_start = list[0].find("label") + 8 journal_name_end = list[0].find("\",\"", journal_name_start) journal_name = list[0][journal_name_start:journal_name_end] journal_name = journal_name.upper() # 查找到的名字也是全大写 msg.msg("journal name", journal_name_raw, "web retrieved", journal_name, "debug", msg.display) return journal_name break except Exception, e: msg.msg("journal name", journal_name, "web retrieved", "retried", "debug", msg.display) msg.msg("journal name", journal_name, "web retrieved", str(e), "error", msg.log) tries -= 1 time.sleep(config.request_refresh_wait)
def adjust_record_number(project, sstr, record_number): # 确定正确的记录数 url = parse_url(project, sstr) tries = config.request_sp_tries # 尝试3次 while(tries > 0): try: opener = requests.Session() content = opener.get(url, timeout=config.request_time_out, headers=agents.get_header()).text # header仍然可以是随机的 max_record_number_start = content.find( "<h3 class=\"result_count left\">Items:") + 37 # 找描述开始地方 max_record_number_end = content.find( '</h3>', max_record_number_start) record_number_str = content[max_record_number_start:max_record_number_end] max_record_number = int(record_number_str.split(" ")[-1]) if max_record_number >= record_number: pass else: record_number = max_record_number msg.msg("record number", "", "changed", str( record_number), "notice", msg.log, msg.display) return record_number break except Exception, e: msg.msg("record number", "", "read", str(e), "error", msg.log) msg.msg("record number", "", "read", "retried", "notice", msg.display) tries -= 1 time.sleep(config.request_refresh_wait)
def crawl_direct(project, sstr): # 用于直接爬sum-page,只能爬第一页,但是不采用phantomjs,速度快 url = parse_url(project, sstr) tries = config.request_sp_tries # 尝试3次 while(tries > 0): try: opener = requests.Session() content = opener.get(url, timeout=config.request_time_out, headers=agents.get_header()).text # header仍然可以是随机的 msg.msg("sum page", "1", "loaded", "proc", "info", msg.display) pmid_list = extract_new_pmid(content) # 提取pmid, 然后排除旧的 if pmid_list: mh.add_new_pmid_many( project, sstr, ut.time_str("full"), "pm", pmid_list) msg.msg("sum page", "1", "loaded", "succ", "info", msg.display, msg.log) break except Exception, e: msg.msg("sum page", "1", "loaded", str(e), "error", msg.log) msg.msg("sum page", "1", "loaded", "retried", "notice", msg.display) tries -= 1 time.sleep(config.request_refresh_wait)
def extract_new_pmid(content): # 从文本中提取pmid的通用办法 pmid_set = [] pmid_raw = re.findall("<dd>\d{8}</dd>", content) for pmid in pmid_raw: pmid = str(pmid[4:-5]) # 去处括号 msg.msg("pmid", str(pmid), "retrieved", "proc", "info", msg.log, msg.display, msg.stat) if pmid not in existed_pmid_set: pmid_set.append(pmid) msg.msg("pmid", str(pmid), "retrieved", "succ", "info", msg.log, msg.display, msg.stat) else: msg.msg("pmid", str(pmid), "skipped", "skip", "info", msg.log, msg.display, msg.stat) return pmid_set
def get_journal_info(ojournal_name, proxy=None): # 查找杂志影响因子、分区, 要求输入精准 url = "http://www.letpub.com.cn/index.php?page=journalapp&view=search" search_str = { "searchname": "", "searchissn": "", "searchfield": "", "searchimpactlow": "", "searchimpacthigh": "", "searchscitype": "", "view": "search", "searchcategory1": "", "searchcategory2": "", "searchjcrkind": "", "searchopenaccess": "", "searchsort": "relevance" } search_str["searchname"] = ojournal_name tries = config.request_dp_tries while tries > 0: try: opener = requests.Session() doc = opener.post(url, timeout=20, data=search_str).text selector = etree.HTML(doc.encode("utf-8")) journal_detail_element = selector.xpath( "//td[@style=\"border:1px #DDD solid; border-collapse:collapse; text-align:left; padding:8px 8px 8px 8px;\"]" ) if len(journal_detail_element): impact_factor = journal_detail_element[2].xpath('string(.)') publication_zone = journal_detail_element[3].xpath( 'string(.)')[0] else: impact_factor = "" publication_zone = "" msg.msg("journal info", ojournal_name, "web retrieved", "succ", "debug", msg.display) return impact_factor, publication_zone break except Exception, e: msg.msg("journal info", ojournal_name, "web retrieved", "retried", "debug", msg.display) msg.msg("journal info", ojournal_name, "web retrieved", str(e), "error", msg.log) tries -= 1 time.sleep(config.request_refresh_wait)
def draw(self): p = self.player d = p.pos[0] + 120 * p.scoreboardside ratio = 0 try: ratio = float(p.hits) / p.arrows_gone_by per = str(format(round(ratio * 100, 2), ".2f")) + "%" except: per = "%" message.msg(self.main.scr, str(p.hits) + "/" + str(p.arrows_gone_by), [d, 10], (0, 0, 255), weight=p.scoreboardside) if p.arrows_gone_by: message.msg(self.main.scr, per, [d, 30], (225 * (1 - ratio), 225 * ratio, 0), size=20, weight=p.scoreboardside) message.msg(self.main.scr, p.points, [d, 100], (0, 0, 200), weight=p.scoreboardside) message.msg(self.main.scr, p.combo, [d, 130], (0, 200, 0), weight=p.scoreboardside) message.msg(self.main.scr, p.misclicks, [d, 160], (200, 0, 0), weight=p.scoreboardside) message.msg(self.main.scr, p.misses, [d, 190], (200, 200, 0), weight=p.scoreboardside) message.msg(self.main.scr, p.totalarrows, [p.pos[0], 20], (100, 200, 230), weight=p.scoreboardside)
if max_record_number >= record_number: pass else: record_number = max_record_number msg.msg("record number", "", "changed", str( record_number), "notice", msg.log, msg.display) return record_number break except Exception, e: msg.msg("record number", "", "read", str(e), "error", msg.log) msg.msg("record number", "", "read", "retried", "notice", msg.display) tries -= 1 time.sleep(config.request_refresh_wait) else: msg.msg("record number", "", "read", "fail", "error", msg.display, msg.log) def extract_new_pmid(content): # 从文本中提取pmid的通用办法 pmid_set = [] pmid_raw = re.findall("<dd>\d{8}</dd>", content) for pmid in pmid_raw: pmid = str(pmid[4:-5]) # 去处括号 msg.msg("pmid", str(pmid), "retrieved", "proc", "info", msg.log, msg.display, msg.stat) if pmid not in existed_pmid_set: pmid_set.append(pmid) msg.msg("pmid", str(pmid), "retrieved", "succ", "info", msg.log, msg.display, msg.stat) else: msg.msg("pmid", str(pmid), "skipped", "skip",
journal_name_end = list[0].find("\",\"", journal_name_start) journal_name = list[0][journal_name_start:journal_name_end] journal_name = journal_name.upper() # 查找到的名字也是全大写 msg.msg("journal name", journal_name_raw, "web retrieved", journal_name, "debug", msg.display) return journal_name break except Exception, e: msg.msg("journal name", journal_name, "web retrieved", "retried", "debug", msg.display) msg.msg("journal name", journal_name, "web retrieved", str(e), "error", msg.log) tries -= 1 time.sleep(config.request_refresh_wait) else: msg.msg("journal name", journal_name, "web retrieved", "fail", "error", msg.log, msg.display) return "" def get_journal_info(ojournal_name, proxy=None): # 查找杂志影响因子、分区, 要求输入精准 url = "http://www.letpub.com.cn/index.php?page=journalapp&view=search" search_str = { "searchname": "", "searchissn": "", "searchfield": "", "searchimpactlow": "", "searchimpacthigh": "", "searchscitype": "", "view": "search", "searchcategory1": "", "searchcategory2": "",
#!/usr/bin/python3 from message import sendmessage, sendmessageicon, sendmessagetitle, msg sendmessage("hello you") sendmessageicon("hello you","face-wink") sendmessagetitle("message","title") msg("message","title","face-angry")
def crawl_detail(pmid, proxy=None): # 爬具体页面 link = "https://www.ncbi.nlm.nih.gov/pubmed/" + str(pmid) tries = config.request_dp_tries # 根据设定重复次数 msg.msg("record", pmid, "retrieved", "proc", "info", msg.display, msg.stat) while (tries > 0): try: authors = [] institues = [] countries = [] flinks = [] opener = requests.Session() # 新建了session保存 content = opener.get( link, timeout=config.request_time_out, headers=agents.get_header()).text # 注意,这里是不断随机换agent的 selector = etree.HTML(content.encode("utf-8")) title_element = selector.xpath( "//div[@class = \"rprt abstract\"]//h1") if len(title_element): title = title_element[0].xpath('string(.)') author_element = selector.xpath("//div[@class = \"auths\"]//a") if len(author_element): for author in author_element: authors.append(author.xpath('string(.)')) journal_element = selector.xpath("//a[@alsec=\"jour\"]/@title") if len(journal_element): journal = journal_element[0] if journal: journal_detail = jn.journal_detail(journal) ojournal = journal_detail[0] journal_if = journal_detail[1] journal_zone = journal_detail[2] abstract_element = selector.xpath( "//*[@id=\"maincontent\"]/div/div[5]/div/div[4]") if len(abstract_element): abstract = abstract_element[0].xpath('string(.)')[8:] key_words_element = selector.xpath( "//*[@id=\"maincontent\"]/div/div[5]/div/div[5]/p") if len(key_words_element): key_words = key_words_element[0].xpath('string(.)').split("; ") else: key_words = [] issue_element = selector.xpath("//div[@class = \"cit\"]") # 年份 if len(issue_element): issue_raw = issue_element[0].xpath('string(.)') issue_start = issue_raw.find(".") issue = issue_raw[issue_start + 2:issue_start + 6] institues_element = selector.xpath("//div[@class=\"afflist\"]//dd") if len(institues_element): for institue in institues_element: institue = institue.xpath('string(.)') institue = ut.regexp_replace( institue, ut.re_email_pm) # 去除pm的email语句 institue = ut.regexp_replace( institue, ut.re_email_general) # 去除所有中间的email institue = institue.replace(" ,", ",") institues.append(institue) institue = institue.replace(", ", ",").replace(".", "") institue_strs = institue.split(",") institue_strs.reverse() # 国家名往往放在最后 i = 0 while i < len(institue_strs): if institue_strs[i] in dictionary.country_names.keys( ): # 如果有这个机构 country_name = dictionary.country_names[ institue_strs[i]] # 直接查询 if not country_name in countries: countries.append(country_name) break else: i += 1 flink_element = selector.xpath( "//div[@class=\"icons portlet\"]//a/@href") if len(flink_element): for flink in flink_element: flinks.append(flink) mh.add_new_content(pmid, title, authors, journal, ojournal, journal_if, journal_zone, issue, abstract, key_words, institues, countries, flinks) msg.msg("record", pmid, "retrieved", "succ", "info", msg.display, msg.stat) break except Exception, e: msg.msg("record", pmid, "retrieved", "retried", "notice", msg.display) msg.msg("record", pmid, "retrieved", str(e), "error", msg.log) tries -= 1 time.sleep(config.request_refresh_wait) # 如果抓不成功,就先休息3秒钟
flinks.append(flink) mh.add_new_content(pmid, title, authors, journal, ojournal, journal_if, journal_zone, issue, abstract, key_words, institues, countries, flinks) msg.msg("record", pmid, "retrieved", "succ", "info", msg.display, msg.stat) break except Exception, e: msg.msg("record", pmid, "retrieved", "retried", "notice", msg.display) msg.msg("record", pmid, "retrieved", str(e), "error", msg.log) tries -= 1 time.sleep(config.request_refresh_wait) # 如果抓不成功,就先休息3秒钟 else: msg.msg("record", pmid, "retrieved", "fail", "error", msg.display, msg.log) return 0 def run_detail_crawler(project, sstr, record_number): pmid_list = get_pmid_list(project, sstr, record_number) pool = Pool(config.detail_crawler_number) # 实例化进程池 pool.map(crawl_detail, pmid_list) pool.close() # 关闭进程池 pool.join() # 等待所有进程结束 if __name__ == '__main__': run_detail_crawler("test", "lactobacillus", 1000)
def save_png(browser): browser.save_screenshot( ut.cur_file_dir() + "/browser/" + ut.time_str("time") + ".png") msg.msg("screenshot", "", "saved", "succ", "debug", msg.display, msg.log)