def login():
"""
Route of login and validations' users
:return: Page of login
"""
if request.method == "POST":
user = request.form.get("username")
password = request.form.get("password")
info_users = Mysql(email=user, user=user, password=password)
if info_users.conf_login():
# Use sessions for validate the access to home
if request.form.get('remember') == "remember":
session.permanent = True
else:
session.permanent = False
session["username"] = info_users.get_user_name()[0]
return redirect(url_for("home"))
else:
# Error of Login
flash("message_alert", "Seu usuário ou senha estão incorretos")
return redirect(url_for("login"))
else:
if "username" in session:
return redirect(url_for("home"))
return render_template("login.html")
def register_user():
"""
:return:Page register user
"""
if request.method == "POST":
email = request.form.get("email")
user = request.form.get("username")
password = request.form.get("password")
info_users = Mysql(email=email, user=user, password=password)
if info_users.register() is True:
flash("message_alert", f"Usuário {user} cadastrado com sucesso")
return redirect(url_for("login"))
else:
flash("message_alert", "Erro ao cadastrar usuário")
return redirect(url_for("register_user"))
return render_template("register.html")
def convert():
db=Mysql()
db.query('select userID from pushinglist group by userID')
dataUID=db.show()
for UID in dataUID:
db.queryparam('select RSSID from pushinglist where userID=%s',UID[0])
dataRID=db.show()
command=ROOT+'/mobi/ day '+ROOT+'/kindlegen/kindlegen'
for RID in dataRID:
db.queryparam('select RSSLink from rsslist where RSSID=%s',RID[0])
dataRLink=db.show()
for Rlink in dataRLink:
command=command+' '+Rlink[0]
# print "python ./dailykindle.py"+' ' +command
# os.system("python ./dailykindle.py"+' '+command)
try:
print "python ./dailykindle.py"+' ' +command
# os.system("python ./dailykindle.py"+' '+command)
except Exception, e:
print str(e)
finally:
import content_per_article
import time
from database import Mysql
connDB1 = Mysql.connDB()
Mysql.exeSearch(connDB1[1])
ids = connDB1[1].fetchall()
p = 0
n = 0
while p <= 8: #循环抓取每一页的内容，实际页数为篇p+1
s = 0
a = content_per_article.get_url_title(p)
if a[0] != []: #判断是否此页的所有文章都在数据库中
while s < content_per_article.x: #循环抓取一页中每篇文章的内容
if (a[0][s],
) in ids or a[7][s] == '': #判断是否有文章已经存在数据库中或者要抓取的文章是否有内容
s += 1
continue
else:
sql = "INSERT INTO xungen(id,title,subtitle,summary,content,picurl,rid,author,create_time,public_time,update_time,isanonymous,content_imgs,city)VALUES"
sql1 = sql + '("' + str(a[0][s]) + '","' + str(
a[1][s]
) + '","' + str(a[2][s]) + '","' + str(a[3][s]) + '","' + str(
a[4][s]) + '","' + str(a[5][s]) + '","' + str(
a[6]) + '","' + str(a[7][s]) + '","' + str(
a[8][s]) + '","' + str(a[9][s]) + '","' + str(
a[10][s]) + '","' + str(a[11]) + '","' + str(
a[12][s]) + '","' + str(a[13]) + '")'
Mysql.exeUpdate(connDB1[0], connDB1[1], sql1)
s += 1
n += 1
def save_user(self, user=None, table_name="user_task"):
if not user:
return
mysql = Mysql()
mysql.connect()
try:
name = user.name.replace('\\', '\\\\').replace("'", "\\'").replace(
':', '\\:')
location = user.location.replace('\\', '\\\\').replace(
"'", "\\'").replace(':', '\\:') if user.location else ''
description = user.description.replace('\\', '\\\\').replace(
"'", "\\'").replace(':', '\\:') if user.description else ''
time_zone = user.time_zone.replace("'",
"\\'") if user.time_zone else ''
utc_offset = user.utc_offset if user.utc_offset else ''
profile_banner_url = user.profile_banner_url if user.profile_banner_url else ''
protected = 1 if user.protected else 0
verified = 1 if user.verified else 0
geo_enabled = 1 if user.geo_enabled else 0
default_profile_image = 1 if user.default_profile_image else 0
sql = """INSERT INTO %s (user_id, screen_name, name, location, created_at, description, statuses_count, friends_count,
followers_count, favourites_count, lang, protected, time_zone, verified, utc_offset, geo_enabled, listed_count,
default_profile_image, profile_background_color, profile_sidebar_fill_color, profile_image_url, profile_banner_url, crawler_date) VALUES
('%s', '%s', '%s', '%s', '%s', '%s', %d, %d, %d, %d, '%s', %d, '%s', %d, '%s', %d, %d, %d,
'%s', '%s', '%s', '%s', '%s')""" % (table_name, user.id, user.screen_name, name, location, user.created_at, description, \
user.statuses_count, user.friends_count, user.followers_count, user.favourites_count, user.lang, protected, time_zone, verified, \
utc_offset, geo_enabled, user.listed_count, default_profile_image, user.profile_background_color, \
user.profile_sidebar_fill_color, user.profile_image_url, profile_banner_url, time.strftime('%Y-%m-%d',time.localtime(time.time())))
sql = sql.encode("utf-8").decode("latin1")
except Exception as e:
print e
mysql.close()
return
try:
mysql.execute(sql)
except Exception as e:
print e
mysql.close()
return
mysql.close()
del user
gc.collect()
def get_url_title(p):
global x #计算每一页有多少篇文章
x = 0
id = []
title = []
subtitle = []
summary = []
content = []
picurl = []
rid = ''
author = []
create_time = []
public_time = []
update_time = []
isanonymous = ''
content_imgs = []
city = ''
Page_con = content_per_page.get_page(p)
connDB1 = Mysql.connDB()
Mysql.exeSearch(connDB1[1])
aids = connDB1[1].fetchall()
for item in Page_con:
res = requests.get(item['link'])
res.encoding = 'utf-8'
soup = BeautifulSoup(res.text, 'lxml')
if (item['aid'], ) in aids or soup.select(
'#post-user') == []: #判断要插入的数据是否已经存在数据库中或者要抓取的内容是否存在
author.append('')
time.sleep(1)
continue
else:
id.append(item['aid'])
title.append(item['title'].replace('"', '”'))
subtitle = title
summary.append(item['digest'].replace('"', '”'))
content.append(
pymysql.escape_string('\n'.join(
str(s) for s in soup.select('p')[2:-6])))
picurl.append(item['cover'])
rid = '0'
author.append(soup.select('#post-user')[0].text)
if soup.select('#post-date') == []:
create_time.append(soup.select('#publish_time')[0].text)
else:
create_time.append(soup.select('#post-date')[0].text)
public_time = create_time
update_time.append(item['update_time'])
isanonymous = 'No'
n = 0
content_img = []
while n < len(soup.select('img')):
if soup.select('img')[n].has_attr("data-src"):
content_img.append(soup.select('img')[n]['data-src'])
n += 1
else:
n += 1
content_imgs.append(';'.join(content_img))
city = 'Macheng'
x += 1
time.sleep(5)
return id, title, subtitle, summary, content, picurl, rid, author, create_time, public_time, update_time, isanonymous, content_imgs, city
def convert():
db = Mysql()
db.query('select userID from pushinglist group by userID')
dataUID = db.show()
for UID in dataUID:
db.queryparam('select RSSID from pushinglist where userID=%s', UID[0])
dataRID = db.show()
command = ROOT + '/mobi/ day ' + ROOT + '/kindlegen/kindlegen'
for RID in dataRID:
db.queryparam('select RSSLink from rsslist where RSSID=%s', RID[0])
dataRLink = db.show()
for Rlink in dataRLink:
command = command + ' ' + Rlink[0]
# print "python ./dailykindle.py"+' ' +command
# os.system("python ./dailykindle.py"+' '+command)
try:
print "python ./dailykindle.py" + ' ' + command
# os.system("python ./dailykindle.py"+' '+command)
except Exception, e:
print str(e)
finally:
from database import Mysql
import backtrader as bt
from datetime import datetime
if __name__ == "__main__":
bao = Baostock.BaostockRebuild()
sh000001 = bao.k_data_query(code="000001.SH", char="date,code,open,high,low,close,volume,amount,adjustflag,turn,pctChg,isST",
start_date="2019-03-10", end_date="2020-01-01", frequency="d", adjustflag="3")
#sh000001.to_csv("Test.csv")
#div_info_test = bao.div_info_query(code="sh.600000", year="2015", yearType="report")
#profitability_info = bao.profitability_info_query(code="sh.600000", year=2017, quarter=2)
#print(profitability_info)
db = Mysql.MysqlConnector(host="1.15.90.134", user="******", passwd="123", auth_plugin="mysql_native_password")
#db.createTable("sh000001")
pdd = db.downloadLastDayData("sh000001")
if pdd.index.size != 0:
print(pdd.head())
del(bao)
#
#
# # 创建策略类
# class SmaCross(bt.Strategy):
# # 定义参数
# params = dict(period=5 # 移动平均期数
# )
#
# # 日志函数
# def log(self, txt, dt=None):
from database import Mysql
mysql = Mysql()
with mysql.connection:
with mysql.connection.cursor() as cursor:
sql = "INSERT INTO `chart_data` (`x`, `y`) VALUES (%s, %s)"
cursor.execute(sql, (1, 1))
sql = "INSERT INTO `chart_data` (`x`, `y`) VALUES (%s, %s)"
cursor.execute(sql, (2, 2))
sql = "INSERT INTO `chart_data` (`x`, `y`) VALUES (%s, %s)"
cursor.execute(sql, (3, 3))
mysql.connection.commit()
print("Finished!")