def auto_fix(self):
if not self.year:
self.year = DateUtil.get_year()
if not self.month:
self.month = DateUtil.get_month()
if not self.day:
self.day = DateUtil.get_day()
def auto_fix(self): if not self.year: self.year = DateUtil.get_year() if not self.month: self.month = DateUtil.get_month() if not self.day: self.day = DateUtil.get_day()
def search():
date = DateUtil.time_to_str(DateUtil.substract_day(DateUtil.now(), 1), Const.STYLE_YYMMDD)
txt = "又到了推荐极具视觉冲击图片的时刻了[微笑][微笑] @当幸福敲不开门 "
url = "http://freepic.wetu.me/preview/%s%s.jpg"
browser = LoginController.get_browser()
passwd_dict = PasswdUtil.get_passwd_dict()
LoginController.mobile_login(browser, passwd_dict[Const.WEIBO_USERNAME], passwd_dict[Const.WEIBO_PASSWD])
service.send_weibo_pics(txt, [url % (date, "i"), url % (date, "p")], browser)
def search():
date = DateUtil.time_to_str(DateUtil.substract_day(DateUtil.now(), 1),
Const.STYLE_YYMMDD)
txt = '又到了推荐极具视觉冲击图片的时刻了[微笑][微笑] @当幸福敲不开门 '
url = 'http://freepic.wetu.me/preview/%s%s.jpg'
browser = LoginController.get_browser()
passwd_dict = PasswdUtil.get_passwd_dict()
LoginController.mobile_login(browser, passwd_dict[Const.WEIBO_USERNAME],
passwd_dict[Const.WEIBO_PASSWD])
service.send_weibo_pics(txt, [url % (date, 'i'), url % (date, 'p')],
browser)
def changeMonth(self):
'''
Changes the month and then copy despesas mensais from previous months if they
don't exist.
'''
from processor.DespesaMensalProcessor import DespesaMensalProcessor
monthFilter = self.month
months = self.monthDAO.find(monthFilter)
month = None
if len(months) == 1:
month = months[0]
else:
month = monthFilter
monthId = self.monthDAO.add(month)
month.id = monthId
print('Copying despesas from the past months...')
DespesaMensalProcessor(month = month).copy()
print('...Done')
despesaMensalProcessor = DespesaMensalProcessor(month = month)
despesaMensalProcessor.updateDespReservaAnual()
despesaMensalProcessor.updateDespCarneLeao()
month.month = DateUtil.toDate(month.month)
return month
def add(self, model):
if model.months:
pMonth = self.month.month
for i in range(model.months):
pagamento = Pagamento(val=model.despesa.val, month=pMonth)
model.pagamentos.append(pagamento)
pMonth = DateUtil.nextMonth(pMonth)
super().add(model)
def forword_by_comment(webdriver, weibo):
print '转发微博,作者：%s, 内容:%s, 发表时间:%s' % (weibo.author_name,weibo.comment, weibo.time)
webdriver.get(weibo.author_url)
time.sleep(4)
weibos = get_all_weibo(webdriver)
if not weibos:
time.sleep(4)
weibos = get_all_weibo(webdriver)
for w in weibos:
try:
ww = weibo_parse(w)
print '候选微博内容:%s, 时间:%s' % (ww.comment, ww.time)
if DateUtil.time_to_str(ww.time, '%Y-%m-%d %H:%M') == DateUtil.time_to_str(weibo.time, '%Y-%m-%d %H:%M'):
forword_weibo(webdriver, w, ww.comment)
return
except:
pass
#如果没找到，则转发第二条，最有可能是
print '没匹配到合适的，转发第二条'
forword_weibo(webdriver, weibos[1], "")
def forword_by_comment(webdriver, weibo):
print '转发微博,作者：%s, 内容:%s, 发表时间:%s' % (weibo.author_name, weibo.comment,
weibo.time)
webdriver.get(weibo.author_url)
time.sleep(4)
weibos = get_all_weibo(webdriver)
if not weibos:
time.sleep(4)
weibos = get_all_weibo(webdriver)
for w in weibos:
try:
ww = weibo_parse(w)
print '候选微博内容:%s, 时间:%s' % (ww.comment, ww.time)
if DateUtil.time_to_str(ww.time,
'%Y-%m-%d %H:%M') == DateUtil.time_to_str(
weibo.time, '%Y-%m-%d %H:%M'):
forword_weibo(webdriver, w, ww.comment)
return
except:
pass
#如果没找到，则转发第二条，最有可能是
print '没匹配到合适的，转发第二条'
forword_weibo(webdriver, weibos[1], "")
def parse_blog(blog_soup):
blog = Blog()
try:
a = blog_soup['class']
blog.is_have_img = 1
except KeyError:
pass
p = blog_soup.find(attrs={"class": "list-top"})
blog.author_name = p.find('a').text.encode('utf-8')
blog.author_url = p.find('a')['href'].encode('utf-8')
blog.time = DateUtil.str_to_time(p.find('span')['data-shared-at'], Const.JIAN_STYLE,)
title_wrap = blog_soup.find(attrs={"class": "title"}).find('a')
blog.title = title_wrap.text.encode('utf-8')
blog.url = title_wrap['href'].encode('utf-8')
nums_str = blog_soup.find(attrs={"class": "list-footer"}).text.encode('utf-8')
pattern = re.compile(r'阅读 ([0-9]+)· 评论 ([0-9]+)· 喜欢 ([0-9]+)')
nums = pattern.search(nums_str).groups()
blog.comment = int(nums[1])
blog.view = int(nums[0])
blog.like = int(nums[2])
return blog
def time_parse(time_str):
if '分钟前' in time_str:
time_str = time_str.split('分钟前')[0]
weibo_time = DateUtil.substract_minus_by_now(int(time_str))
elif '今天 ' in time_str:
time_str = time_str.split('今天 ')[1]
weibo_style = '%Y-%m-%d '
day = DateUtil.now_format(weibo_style)
time = day + time_str + ':00'
weibo_time = DateUtil.str_to_time(time, TimeConst.Const.STYLE_MYSQL)
elif '2015' in time_str or '2014' in time_str or '2013' in time_str or '2012' in time_str or '2011' in time_str:
style = "%Y-%m-%d %H:%M"
weibo_time = DateUtil.str_to_time(time_str, style)
elif '秒前' in time_str:
style = "%Y-%m-%d %H:%M"
weibo_time = DateUtil.now_format(style)
else:
weibo_style = '%Y年%m月%d日 %H:%M:%S'
time_str = str(DateUtil.now().year) + '年' + time_str + ':00'
weibo_time = DateUtil.str_to_time(time_str, weibo_style)
return weibo_time
def time_parse(time_str):
if '分钟前' in time_str:
time_str = time_str.split('分钟前')[0]
weibo_time = DateUtil.substract_minus_by_now(int(time_str))
elif '今天 ' in time_str:
time_str = time_str.split('今天 ')[1]
weibo_style = '%Y-%m-%d '
day = DateUtil.now_format(weibo_style)
time = day + time_str + ':00'
weibo_time = DateUtil.str_to_time(time, TimeConst.Const.STYLE_MYSQL)
elif '2015' in time_str or '2014' in time_str or '2013' in time_str or '2012' in time_str or '2011' in time_str:
style = "%Y-%m-%d %H:%M"
weibo_time = DateUtil.str_to_time(time_str, style)
elif '秒前' in time_str:
style = "%Y-%m-%d %H:%M"
weibo_time = DateUtil.now_format(style)
else:
weibo_style = '%Y年%m月%d日 %H:%M:%S'
time_str = str(DateUtil.now().year) + '年' + time_str + ':00'
weibo_time = DateUtil.str_to_time(time_str, weibo_style)
return weibo_time
def get_achievement(self, req_id):
stocks, numbers = self.invest_dao.get_stock_position(req_id)
ratios = np.array(numbers) / np.array(numbers).sum()
start_date = date(2016, 9, 1)
end_date = date(2018, 6, 1)
stocks_price = []
for stock in stocks:
_, values = self.stock_dao.get_stock_period_price(
stock, start_date, end_date)
for i in range(values.__len__()):
if np.isnan(values[i]):
values[i] = 0
stocks_price.append(values)
myfund = []
for i in range(stocks_price[0].__len__()):
fund = 0
for j in range(stocks_price.__len__()):
fund += stocks_price[j][i] * ratios[j]
myfund.append(fund)
base_300 = self.transform_list_with_date(
self.stock_dao.get_csi_300(start_date, end_date))
base_300g = [0]
myfundg = [0]
for i in range(base_300.__len__() - 1):
base_300g.append((base_300[i + 1] - base_300[i]) / base_300[i])
for i in range(myfund.__len__() - 1):
myfundg.append((myfund[i + 1] - myfund[i]) / myfund[i])
revenueg = []
for i in range(myfundg.__len__()):
revenueg.append(myfundg[i] - base_300g[i])
revenuem = []
for i in range(np.math.floor(revenueg.__len__() / 30)):
index = i * 30
month_revenueg = np.average(myfundg[index:index + 30])
revenuem.append(month_revenueg)
revenueq = []
for i in range(np.math.floor(revenueg.__len__() / 90)):
index = i * 90
quarter_revenueg = np.average(myfundg[index:index + 90])
revenueq.append(quarter_revenueg)
revenue = []
for i in range(myfund.__len__()):
revenue.append(myfund[i] - base_300[i])
back = []
for i in range(revenue.__len__()):
max_revenue = max(revenue[0:i + 1])
value = (revenue[i] - max_revenue) / max_revenue
if value < 0:
value = 0
back.append(value)
# 开始设置返回值
chart_data = []
for i in range(revenueg.__len__()):
chart_data.append({
'date':
DateUtil.get_format_date(start_date + timedelta(i)),
'returnNum':
"%.2f" % revenueg[i],
'revenueNum':
"%.2f" % back[i]
})
form_data = []
for j in range(stocks.__len__()):
stock_price = self.stock_dao.get_stock_period_price(
stocks[j], start_date, end_date)[1]
stock_priceg = [0]
for i in range(myfund.__len__() - 1):
stock_priceg.append(
(stock_price[i + 1] - stock_price[i]) / stock_price[i])
stock_revenueg = []
for i in range(stock_priceg.__len__()):
stock_revenueg.append(stock_priceg[i] - base_300g[i])
stock_revenuem = []
for i in range(np.math.floor(stock_revenueg.__len__() / 30)):
index = i * 30
month_stock_revenueg = np.average(stock_revenueg[index:index +
30])
stock_revenuem.append(month_stock_revenueg)
stock_winm_num = 0
for i in range(stock_revenuem.__len__()):
if stock_revenuem[i] > 0:
stock_winm_num += 0
stock_winm = stock_winm_num * 1.0 / stock_revenuem.__len__()
stock_revenue = []
for i in range(stock_price.__len__()):
stock_revenue.append(stock_price[i] - base_300[i])
stock_back = []
for i in range(stock_revenue.__len__()):
max_revenue = max(stock_revenue[0:i + 1])
value = (stock_revenue[i] - max_revenue) / max_revenue
if value < 0:
value = 0
stock_back.append(value)
stock_max_back, stock_max_back_start_date, stock_max_back_end_date, stock_max_back_return_time = self.calculate_max_back(
stock_back) # 超额最大回撤,超额最大回撤起开始日,超额最大回撤起终止日,超额最大回撤回补日
form_data.append({
'quotaName':
stocks[j], # 名称
'period':
7, # 周期数：月
'totalRevenue':
stock_max_back, # 超额累计回撤率：百分比
'monthly':
stock_winm, # 月胜率：百分比
'maxReturn':
"%.2f" % sum(stock_revenueg), # 超额最大回撤率：百分比
'maxReturnBegin':
DateUtil.get_format_date(
stock_max_back_start_date), # 超额最大回撤起始日期：date
'maxReturnEnd':
DateUtil.get_format_date(
stock_max_back_end_date), # 超额最大回撤结束日期：date
'maxReturnSupplement':
stock_max_back_return_time # 超额最大回撤补期：天
})
return {'chartData': chart_data, 'formData': form_data}
def __zldate_parse(self, date):
date_str = '2016-'+date
return DateUtil.str_to_time(date_str, '%Y-%m-%d')
html = 'http://search.51job.com/jobsearch/search_result.php?jobarea=070200%2C00&keyword=java&curr_page=' + str(page)
response = HttpUtil.opener_request(html, opener)
soup = BeautifulSoup(response)
job_list = soup.find(id='resultList').findAll('div' ,attrs={"class": "el"})[1:]
self.__insert_jobs(map(self.__get_51job_detail, job_list))
def __get_51job_detail(self, job_soup):
job = Job()
job.job = job_soup.find('p' ,attrs={"class": "t1"}).text
job.company_name = job_soup.find('span' ,attrs={"class": "t2"}).text
job.publish_date = self.__zldate_parse(job_soup.find('span' ,attrs={"class": "t5"}).text)
job.source = '51'
return job
def __zldate_parse(self, date):
date_str = '2016-'+date
return DateUtil.str_to_time(date_str, '%Y-%m-%d')
def __insert_jobs(self, jobs):
for i,job in enumerate(jobs):
if job.publish_date <self.date:
dao.insert_jobs(jobs[:i-1])
raise Exception
dao.insert_jobs(jobs)
if __name__ == '__main__':
date = DateUtil.substract_day(DateUtil.now(), 1)
crawler = FOCrawler(date)
crawler.sample()
