def search():
date = DateUtil.time_to_str(DateUtil.substract_day(DateUtil.now(), 1), Const.STYLE_YYMMDD)
txt = "又到了推荐极具视觉冲击图片的时刻了[微笑][微笑] @当幸福敲不开门 "
url = "http://freepic.wetu.me/preview/%s%s.jpg"
browser = LoginController.get_browser()
passwd_dict = PasswdUtil.get_passwd_dict()
LoginController.mobile_login(browser, passwd_dict[Const.WEIBO_USERNAME], passwd_dict[Const.WEIBO_PASSWD])
service.send_weibo_pics(txt, [url % (date, "i"), url % (date, "p")], browser)
def search():
date = DateUtil.time_to_str(DateUtil.substract_day(DateUtil.now(), 1),
Const.STYLE_YYMMDD)
txt = '又到了推荐极具视觉冲击图片的时刻了[微笑][微笑] @当幸福敲不开门 '
url = 'http://freepic.wetu.me/preview/%s%s.jpg'
browser = LoginController.get_browser()
passwd_dict = PasswdUtil.get_passwd_dict()
LoginController.mobile_login(browser, passwd_dict[Const.WEIBO_USERNAME],
passwd_dict[Const.WEIBO_PASSWD])
service.send_weibo_pics(txt, [url % (date, 'i'), url % (date, 'p')],
browser)
def time_parse(time_str):
if '分钟前' in time_str:
time_str = time_str.split('分钟前')[0]
weibo_time = DateUtil.substract_minus_by_now(int(time_str))
elif '今天 ' in time_str:
time_str = time_str.split('今天 ')[1]
weibo_style = '%Y-%m-%d '
day = DateUtil.now_format(weibo_style)
time = day + time_str + ':00'
weibo_time = DateUtil.str_to_time(time, TimeConst.Const.STYLE_MYSQL)
elif '2015' in time_str or '2014' in time_str or '2013' in time_str or '2012' in time_str or '2011' in time_str:
style = "%Y-%m-%d %H:%M"
weibo_time = DateUtil.str_to_time(time_str, style)
elif '秒前' in time_str:
style = "%Y-%m-%d %H:%M"
weibo_time = DateUtil.now_format(style)
else:
weibo_style = '%Y年%m月%d日 %H:%M:%S'
time_str = str(DateUtil.now().year) + '年' + time_str + ':00'
weibo_time = DateUtil.str_to_time(time_str, weibo_style)
return weibo_time
def time_parse(time_str):
if '分钟前' in time_str:
time_str = time_str.split('分钟前')[0]
weibo_time = DateUtil.substract_minus_by_now(int(time_str))
elif '今天 ' in time_str:
time_str = time_str.split('今天 ')[1]
weibo_style = '%Y-%m-%d '
day = DateUtil.now_format(weibo_style)
time = day + time_str + ':00'
weibo_time = DateUtil.str_to_time(time, TimeConst.Const.STYLE_MYSQL)
elif '2015' in time_str or '2014' in time_str or '2013' in time_str or '2012' in time_str or '2011' in time_str:
style = "%Y-%m-%d %H:%M"
weibo_time = DateUtil.str_to_time(time_str, style)
elif '秒前' in time_str:
style = "%Y-%m-%d %H:%M"
weibo_time = DateUtil.now_format(style)
else:
weibo_style = '%Y年%m月%d日 %H:%M:%S'
time_str = str(DateUtil.now().year) + '年' + time_str + ':00'
weibo_time = DateUtil.str_to_time(time_str, weibo_style)
return weibo_time
html = 'http://search.51job.com/jobsearch/search_result.php?jobarea=070200%2C00&keyword=java&curr_page=' + str(page)
response = HttpUtil.opener_request(html, opener)
soup = BeautifulSoup(response)
job_list = soup.find(id='resultList').findAll('div' ,attrs={"class": "el"})[1:]
self.__insert_jobs(map(self.__get_51job_detail, job_list))
def __get_51job_detail(self, job_soup):
job = Job()
job.job = job_soup.find('p' ,attrs={"class": "t1"}).text
job.company_name = job_soup.find('span' ,attrs={"class": "t2"}).text
job.publish_date = self.__zldate_parse(job_soup.find('span' ,attrs={"class": "t5"}).text)
job.source = '51'
return job
def __zldate_parse(self, date):
date_str = '2016-'+date
return DateUtil.str_to_time(date_str, '%Y-%m-%d')
def __insert_jobs(self, jobs):
for i,job in enumerate(jobs):
if job.publish_date <self.date:
dao.insert_jobs(jobs[:i-1])
raise Exception
dao.insert_jobs(jobs)
if __name__ == '__main__':
date = DateUtil.substract_day(DateUtil.now(), 1)
crawler = FOCrawler(date)
crawler.sample()
