def search(): date = DateUtil.time_to_str(DateUtil.substract_day(DateUtil.now(), 1), Const.STYLE_YYMMDD) txt = "又到了推荐极具视觉冲击图片的时刻了[微笑][微笑] @当幸福敲不开门 " url = "http://freepic.wetu.me/preview/%s%s.jpg" browser = LoginController.get_browser() passwd_dict = PasswdUtil.get_passwd_dict() LoginController.mobile_login(browser, passwd_dict[Const.WEIBO_USERNAME], passwd_dict[Const.WEIBO_PASSWD]) service.send_weibo_pics(txt, [url % (date, "i"), url % (date, "p")], browser)
def search(): date = DateUtil.time_to_str(DateUtil.substract_day(DateUtil.now(), 1), Const.STYLE_YYMMDD) txt = '又到了推荐极具视觉冲击图片的时刻了[微笑][微笑] @当幸福敲不开门 ' url = 'http://freepic.wetu.me/preview/%s%s.jpg' browser = LoginController.get_browser() passwd_dict = PasswdUtil.get_passwd_dict() LoginController.mobile_login(browser, passwd_dict[Const.WEIBO_USERNAME], passwd_dict[Const.WEIBO_PASSWD]) service.send_weibo_pics(txt, [url % (date, 'i'), url % (date, 'p')], browser)
def time_parse(time_str): if '分钟前' in time_str: time_str = time_str.split('分钟前')[0] weibo_time = DateUtil.substract_minus_by_now(int(time_str)) elif '今天 ' in time_str: time_str = time_str.split('今天 ')[1] weibo_style = '%Y-%m-%d ' day = DateUtil.now_format(weibo_style) time = day + time_str + ':00' weibo_time = DateUtil.str_to_time(time, TimeConst.Const.STYLE_MYSQL) elif '2015' in time_str or '2014' in time_str or '2013' in time_str or '2012' in time_str or '2011' in time_str: style = "%Y-%m-%d %H:%M" weibo_time = DateUtil.str_to_time(time_str, style) elif '秒前' in time_str: style = "%Y-%m-%d %H:%M" weibo_time = DateUtil.now_format(style) else: weibo_style = '%Y年%m月%d日 %H:%M:%S' time_str = str(DateUtil.now().year) + '年' + time_str + ':00' weibo_time = DateUtil.str_to_time(time_str, weibo_style) return weibo_time
html = 'http://search.51job.com/jobsearch/search_result.php?jobarea=070200%2C00&keyword=java&curr_page=' + str(page) response = HttpUtil.opener_request(html, opener) soup = BeautifulSoup(response) job_list = soup.find(id='resultList').findAll('div' ,attrs={"class": "el"})[1:] self.__insert_jobs(map(self.__get_51job_detail, job_list)) def __get_51job_detail(self, job_soup): job = Job() job.job = job_soup.find('p' ,attrs={"class": "t1"}).text job.company_name = job_soup.find('span' ,attrs={"class": "t2"}).text job.publish_date = self.__zldate_parse(job_soup.find('span' ,attrs={"class": "t5"}).text) job.source = '51' return job def __zldate_parse(self, date): date_str = '2016-'+date return DateUtil.str_to_time(date_str, '%Y-%m-%d') def __insert_jobs(self, jobs): for i,job in enumerate(jobs): if job.publish_date <self.date: dao.insert_jobs(jobs[:i-1]) raise Exception dao.insert_jobs(jobs) if __name__ == '__main__': date = DateUtil.substract_day(DateUtil.now(), 1) crawler = FOCrawler(date) crawler.sample()