def auto_fix(self): if not self.year: self.year = DateUtil.get_year() if not self.month: self.month = DateUtil.get_month() if not self.day: self.day = DateUtil.get_day()
def search(): date = DateUtil.time_to_str(DateUtil.substract_day(DateUtil.now(), 1), Const.STYLE_YYMMDD) txt = "又到了推荐极具视觉冲击图片的时刻了[微笑][微笑] @当幸福敲不开门 " url = "http://freepic.wetu.me/preview/%s%s.jpg" browser = LoginController.get_browser() passwd_dict = PasswdUtil.get_passwd_dict() LoginController.mobile_login(browser, passwd_dict[Const.WEIBO_USERNAME], passwd_dict[Const.WEIBO_PASSWD]) service.send_weibo_pics(txt, [url % (date, "i"), url % (date, "p")], browser)
def search(): date = DateUtil.time_to_str(DateUtil.substract_day(DateUtil.now(), 1), Const.STYLE_YYMMDD) txt = '又到了推荐极具视觉冲击图片的时刻了[微笑][微笑] @当幸福敲不开门 ' url = 'http://freepic.wetu.me/preview/%s%s.jpg' browser = LoginController.get_browser() passwd_dict = PasswdUtil.get_passwd_dict() LoginController.mobile_login(browser, passwd_dict[Const.WEIBO_USERNAME], passwd_dict[Const.WEIBO_PASSWD]) service.send_weibo_pics(txt, [url % (date, 'i'), url % (date, 'p')], browser)
def changeMonth(self): ''' Changes the month and then copy despesas mensais from previous months if they don't exist. ''' from processor.DespesaMensalProcessor import DespesaMensalProcessor monthFilter = self.month months = self.monthDAO.find(monthFilter) month = None if len(months) == 1: month = months[0] else: month = monthFilter monthId = self.monthDAO.add(month) month.id = monthId print('Copying despesas from the past months...') DespesaMensalProcessor(month = month).copy() print('...Done') despesaMensalProcessor = DespesaMensalProcessor(month = month) despesaMensalProcessor.updateDespReservaAnual() despesaMensalProcessor.updateDespCarneLeao() month.month = DateUtil.toDate(month.month) return month
def add(self, model): if model.months: pMonth = self.month.month for i in range(model.months): pagamento = Pagamento(val=model.despesa.val, month=pMonth) model.pagamentos.append(pagamento) pMonth = DateUtil.nextMonth(pMonth) super().add(model)
def forword_by_comment(webdriver, weibo): print '转发微博,作者:%s, 内容:%s, 发表时间:%s' % (weibo.author_name,weibo.comment, weibo.time) webdriver.get(weibo.author_url) time.sleep(4) weibos = get_all_weibo(webdriver) if not weibos: time.sleep(4) weibos = get_all_weibo(webdriver) for w in weibos: try: ww = weibo_parse(w) print '候选微博内容:%s, 时间:%s' % (ww.comment, ww.time) if DateUtil.time_to_str(ww.time, '%Y-%m-%d %H:%M') == DateUtil.time_to_str(weibo.time, '%Y-%m-%d %H:%M'): forword_weibo(webdriver, w, ww.comment) return except: pass #如果没找到,则转发第二条,最有可能是 print '没匹配到合适的,转发第二条' forword_weibo(webdriver, weibos[1], "")
def forword_by_comment(webdriver, weibo): print '转发微博,作者:%s, 内容:%s, 发表时间:%s' % (weibo.author_name, weibo.comment, weibo.time) webdriver.get(weibo.author_url) time.sleep(4) weibos = get_all_weibo(webdriver) if not weibos: time.sleep(4) weibos = get_all_weibo(webdriver) for w in weibos: try: ww = weibo_parse(w) print '候选微博内容:%s, 时间:%s' % (ww.comment, ww.time) if DateUtil.time_to_str(ww.time, '%Y-%m-%d %H:%M') == DateUtil.time_to_str( weibo.time, '%Y-%m-%d %H:%M'): forword_weibo(webdriver, w, ww.comment) return except: pass #如果没找到,则转发第二条,最有可能是 print '没匹配到合适的,转发第二条' forword_weibo(webdriver, weibos[1], "")
def parse_blog(blog_soup): blog = Blog() try: a = blog_soup['class'] blog.is_have_img = 1 except KeyError: pass p = blog_soup.find(attrs={"class": "list-top"}) blog.author_name = p.find('a').text.encode('utf-8') blog.author_url = p.find('a')['href'].encode('utf-8') blog.time = DateUtil.str_to_time(p.find('span')['data-shared-at'], Const.JIAN_STYLE,) title_wrap = blog_soup.find(attrs={"class": "title"}).find('a') blog.title = title_wrap.text.encode('utf-8') blog.url = title_wrap['href'].encode('utf-8') nums_str = blog_soup.find(attrs={"class": "list-footer"}).text.encode('utf-8') pattern = re.compile(r'阅读 ([0-9]+)· 评论 ([0-9]+)· 喜欢 ([0-9]+)') nums = pattern.search(nums_str).groups() blog.comment = int(nums[1]) blog.view = int(nums[0]) blog.like = int(nums[2]) return blog
def time_parse(time_str): if '分钟前' in time_str: time_str = time_str.split('分钟前')[0] weibo_time = DateUtil.substract_minus_by_now(int(time_str)) elif '今天 ' in time_str: time_str = time_str.split('今天 ')[1] weibo_style = '%Y-%m-%d ' day = DateUtil.now_format(weibo_style) time = day + time_str + ':00' weibo_time = DateUtil.str_to_time(time, TimeConst.Const.STYLE_MYSQL) elif '2015' in time_str or '2014' in time_str or '2013' in time_str or '2012' in time_str or '2011' in time_str: style = "%Y-%m-%d %H:%M" weibo_time = DateUtil.str_to_time(time_str, style) elif '秒前' in time_str: style = "%Y-%m-%d %H:%M" weibo_time = DateUtil.now_format(style) else: weibo_style = '%Y年%m月%d日 %H:%M:%S' time_str = str(DateUtil.now().year) + '年' + time_str + ':00' weibo_time = DateUtil.str_to_time(time_str, weibo_style) return weibo_time
def get_achievement(self, req_id): stocks, numbers = self.invest_dao.get_stock_position(req_id) ratios = np.array(numbers) / np.array(numbers).sum() start_date = date(2016, 9, 1) end_date = date(2018, 6, 1) stocks_price = [] for stock in stocks: _, values = self.stock_dao.get_stock_period_price( stock, start_date, end_date) for i in range(values.__len__()): if np.isnan(values[i]): values[i] = 0 stocks_price.append(values) myfund = [] for i in range(stocks_price[0].__len__()): fund = 0 for j in range(stocks_price.__len__()): fund += stocks_price[j][i] * ratios[j] myfund.append(fund) base_300 = self.transform_list_with_date( self.stock_dao.get_csi_300(start_date, end_date)) base_300g = [0] myfundg = [0] for i in range(base_300.__len__() - 1): base_300g.append((base_300[i + 1] - base_300[i]) / base_300[i]) for i in range(myfund.__len__() - 1): myfundg.append((myfund[i + 1] - myfund[i]) / myfund[i]) revenueg = [] for i in range(myfundg.__len__()): revenueg.append(myfundg[i] - base_300g[i]) revenuem = [] for i in range(np.math.floor(revenueg.__len__() / 30)): index = i * 30 month_revenueg = np.average(myfundg[index:index + 30]) revenuem.append(month_revenueg) revenueq = [] for i in range(np.math.floor(revenueg.__len__() / 90)): index = i * 90 quarter_revenueg = np.average(myfundg[index:index + 90]) revenueq.append(quarter_revenueg) revenue = [] for i in range(myfund.__len__()): revenue.append(myfund[i] - base_300[i]) back = [] for i in range(revenue.__len__()): max_revenue = max(revenue[0:i + 1]) value = (revenue[i] - max_revenue) / max_revenue if value < 0: value = 0 back.append(value) # 开始设置返回值 chart_data = [] for i in range(revenueg.__len__()): chart_data.append({ 'date': DateUtil.get_format_date(start_date + timedelta(i)), 'returnNum': "%.2f" % revenueg[i], 'revenueNum': "%.2f" % back[i] }) form_data = [] for j in range(stocks.__len__()): stock_price = self.stock_dao.get_stock_period_price( stocks[j], start_date, end_date)[1] stock_priceg = [0] for i in range(myfund.__len__() - 1): stock_priceg.append( (stock_price[i + 1] - stock_price[i]) / stock_price[i]) stock_revenueg = [] for i in range(stock_priceg.__len__()): stock_revenueg.append(stock_priceg[i] - base_300g[i]) stock_revenuem = [] for i in range(np.math.floor(stock_revenueg.__len__() / 30)): index = i * 30 month_stock_revenueg = np.average(stock_revenueg[index:index + 30]) stock_revenuem.append(month_stock_revenueg) stock_winm_num = 0 for i in range(stock_revenuem.__len__()): if stock_revenuem[i] > 0: stock_winm_num += 0 stock_winm = stock_winm_num * 1.0 / stock_revenuem.__len__() stock_revenue = [] for i in range(stock_price.__len__()): stock_revenue.append(stock_price[i] - base_300[i]) stock_back = [] for i in range(stock_revenue.__len__()): max_revenue = max(stock_revenue[0:i + 1]) value = (stock_revenue[i] - max_revenue) / max_revenue if value < 0: value = 0 stock_back.append(value) stock_max_back, stock_max_back_start_date, stock_max_back_end_date, stock_max_back_return_time = self.calculate_max_back( stock_back) # 超额最大回撤,超额最大回撤起开始日,超额最大回撤起终止日,超额最大回撤回补日 form_data.append({ 'quotaName': stocks[j], # 名称 'period': 7, # 周期数:月 'totalRevenue': stock_max_back, # 超额累计回撤率:百分比 'monthly': stock_winm, # 月胜率:百分比 'maxReturn': "%.2f" % sum(stock_revenueg), # 超额最大回撤率:百分比 'maxReturnBegin': DateUtil.get_format_date( stock_max_back_start_date), # 超额最大回撤起始日期:date 'maxReturnEnd': DateUtil.get_format_date( stock_max_back_end_date), # 超额最大回撤结束日期:date 'maxReturnSupplement': stock_max_back_return_time # 超额最大回撤补期:天 }) return {'chartData': chart_data, 'formData': form_data}
def __zldate_parse(self, date): date_str = '2016-'+date return DateUtil.str_to_time(date_str, '%Y-%m-%d')
html = 'http://search.51job.com/jobsearch/search_result.php?jobarea=070200%2C00&keyword=java&curr_page=' + str(page) response = HttpUtil.opener_request(html, opener) soup = BeautifulSoup(response) job_list = soup.find(id='resultList').findAll('div' ,attrs={"class": "el"})[1:] self.__insert_jobs(map(self.__get_51job_detail, job_list)) def __get_51job_detail(self, job_soup): job = Job() job.job = job_soup.find('p' ,attrs={"class": "t1"}).text job.company_name = job_soup.find('span' ,attrs={"class": "t2"}).text job.publish_date = self.__zldate_parse(job_soup.find('span' ,attrs={"class": "t5"}).text) job.source = '51' return job def __zldate_parse(self, date): date_str = '2016-'+date return DateUtil.str_to_time(date_str, '%Y-%m-%d') def __insert_jobs(self, jobs): for i,job in enumerate(jobs): if job.publish_date <self.date: dao.insert_jobs(jobs[:i-1]) raise Exception dao.insert_jobs(jobs) if __name__ == '__main__': date = DateUtil.substract_day(DateUtil.now(), 1) crawler = FOCrawler(date) crawler.sample()