def queryMusicsM(reqJson):
pageno = reqJson['pageno']
pagesize = reqJson['pagesize']
resJson = dict()
resJson['flag'] = "error"
resJson['msg'] = "查询失败"
resJson['data'] = []
try:
query = session.query(MusicsM).limit(pageno).offset(pagesize).all()
if len(query) != 0:
resJson['flag'] = "success"
resJson['msg'] = "登录成功"
data = []
for item in query:
data.append(item.to_json())
resJson['data'] = data
except InvalidRequestError:
session.rollback()
resJson['msg'] = 'InvalidRequestError %r' % repr(InvalidRequestError)
logging.error('InvalidRequestError %r' % repr(InvalidRequestError))
except Exception as err:
session.rollback()
resJson['msg'] = 'Error %r' % repr(err)
logging.error('Error %r' % repr(err))
return resJson
def scrapyDetail(self, detailurl, proxies, id):
headers = {
'Connection': 'close',
"user-agent": userAgents[random.randrange(0, len(userAgents))]
}
req = s.get(detailurl, headers=headers, proxies=proxies, timeout=2)
htmlStr = req.text
ele = etree.HTML(htmlStr)
content_ele = ele.xpath("//div[@class='post-content']/div")[0].xpath(
"p")
content_str = "<div>"
for ele_item in content_ele:
content_str = content_str + str(etree.tostring(
ele_item, encoding="utf-8", method="HTML", pretty_print=True),
encoding="utf8")
content_str + '</div>'
try:
Photography = session.query(PhotographyM).filter(
PhotographyM.id == id).all()
if len(Photography) != 0:
Photography[0].content = content_str
session.commit()
except InvalidRequestError:
print("更新Error: %r" % InvalidRequestError)
session.rollback()
except Exception as err:
print("Mysql2未知Error: %r" % err)
session.rollback()
def deleteMusicsM(reqJson):
filmId = reqJson['filmId']
resJson = dict()
resJson['flag'] = "error"
resJson['msg'] = "删除失败"
if filmId == "" and not filmId:
logging.warn("FilmId warn:FilmId不存在")
print("FilmId warn:FilmId不存在")
resJson['msg'] = "FilmId warn:FilmId不存在"
else:
try:
session.query(MusicsM).filter(MusicsM.id == filmId).delete(
synchronize_session=False)
resJson['flag'] = "success"
resJson['msg'] = "删除成功"
except InvalidRequestError:
session.rollback()
resJson['msg'] = "InvalidRequestError:%r" % repr(
InvalidRequestError)
logging.error("InvalidRequestError:%r" % repr(InvalidRequestError))
except Exception as err:
session.rollback()
resJson['msg'] = "Error:%r" % repr(err)
logging.error("Error %r" % repr(err))
return resJson
def lookDetailFilmsM(reqJson):
filmId = reqJson['filmId']
resJson = {}
resJson['flag'] = "error"
resJson['msg'] = "查询失败"
if filmId == "" and not filmId:
logging.warn("FilmId warn:FilmId不存在")
print("FilmId warn:FilmId不存在")
resJson['msg'] = "FilmId warn:FilmId不存在"
else:
try:
detailFilms = session.query(FilmsM).filter(
FilmsM.id == filmId).all()
resJson['flag'] = "success"
resJson['msg'] = "查询成功"
if len(detailFilms) != 0:
resJson['data'] = detailFilms[0].to_json()
except InvalidRequestError:
session.rollback()
resJson['msg'] = "InvalidRequestError:%r" % repr(
InvalidRequestError)
logging.error("InvalidRequestError:%r" % repr(InvalidRequestError))
except Exception as err:
session.rollback()
resJson['msg'] = "Error:%r" % repr(err)
logging.error("Error %r" % repr(err))
return resJson
def savePic(self, imgcover, proxies, id):
basedir = os.path.dirname(__file__)
hostpath = parse.urlparse(imgcover)
imgpathlist = hostpath[4].split("/")
imgname = imgpathlist[len(imgpathlist) - 1].split("&")[0]
imgpath = os.path.abspath(
os.path.join(basedir, '..', '..', 'static', 'photography',
"/".join(imgpathlist[3:5])))
headers = {
'Connection': 'close',
"user-agent": userAgents[random.randrange(0, len(userAgents))]
}
if not os.path.exists(imgpath):
print("路径不存在,正在创建路径~~~~~~")
os.makedirs(imgpath)
try:
res = s.get(imgcover, headers=headers, proxies=proxies, timeout=2)
with open(os.path.join(imgpath, imgname), "wb") as fp:
fp.write(res.content)
fp.close()
try:
query = session.query(PhotographyM).filter(
PhotographyM.id == id).all()[0]
imgurl = '/static/photography/{0}/{1}'.format(
"/".join(imgpathlist[3:5]), imgname)
query.imgurlstr = imgurl
session.commit()
except InvalidRequestError as err:
print("InvalidRequestError %r" % repr(err))
session.rollback()
except Exception as e:
print("Exception %r" % repr(e))
session.rollback()
except Exception as err:
print("图片下载Error:{0}".format(err))
def isLogin(reqJson):
uname = reqJson['uname']
pwd = reqJson['pwd']
resJson = dict()
resJson['flag'] = "error"
resJson['msg'] = "登录失败"
resJson['data'] = {}
try:
query = session.query(UsersM).filter(
and_(UsersM.uname == uname, UsersM.pwd == pwd)).all()
if len(query) != 0:
resJson['flag'] = "success"
resJson['msg'] = "登录成功"
q_menu = session.query(MenusM).all()
if len(q_menu) != 0:
data = []
for item in q_menu:
data.append(item.to_json())
resJson['data'] = {"menus": data}
except InvalidRequestError:
session.rollback()
resJson['msg'] = "InvalidRequestError:%r" % repr(InvalidRequestError)
logging.error("InvalidRequestError:%r" % repr(InvalidRequestError))
except Exception as err:
session.rollback()
resJson['msg'] = "Exception:%r" % repr(Exception)
logging.error("Exception:%r" % repr(Exception))
return resJson
def saveMysql(self, title, author, desc):
try:
insert_sql = PhotographyM(title, author, desc)
session.add(insert_sql)
session.commit()
return insert_sql.id
except InvalidRequestError:
print("插入Error: %r" % InvalidRequestError)
session.rollback()
except Exception as err:
print("Mysql3未知Error: %r" % err)
session.rollback()
def queryFilmsM(reqJson):
pageno = reqJson['pageno']
pagesize = reqJson['pagesize']
resJson = dict()
resJson['flag'] = "error"
resJson['msg'] = "查询失败"
resJson['data'] = []
try:
offsetnum = pagesize * (pageno - 1)
total = session.query(func.count(FilmsM.id)).scalar()
if offsetnum < total:
query = session.query(
FilmsM.id, FilmsM.title, FilmsM.type, FilmsM.score,
FilmsM.fimgurl).limit(pagesize).offset(offsetnum).all()
resJson['flag'] = "success"
resJson['msg'] = "查询成功"
if len(query) != 0:
data = []
for item in query:
obj = {
"id": item[0],
"title": item[1],
"type": item[2],
"score": item[3],
"fimgurl": item[4]
}
data.append(obj)
resJson['data'] = data
else:
resJson['flag'] = "success"
resJson['msg'] = "查询成功"
resJson['data'] = []
except InvalidRequestError:
session.rollback()
resJson['msg'] = 'InvalidRequestError %r' % repr(InvalidRequestError)
logging.error('InvalidRequestError %r' % repr(InvalidRequestError))
except Exception as err:
session.rollback()
resJson['msg'] = 'Error %r' % repr(err)
logging.error('Error %r' % repr(err))
return resJson
def scrapyDetail(self, url, proxies):
headers = {
"user-agent": userAgents[random.randrange(0, len(userAgents))]
}
try:
urlStr = url + "?tag=热门&from=gaia_video"
res_html = requests.get(urlStr,
headers=headers,
proxies=proxies,
timeout=2).text
res_html = res_html.encode("utf-8", 'ignore')
ele = etree.HTML(res_html)
title = ele.xpath('//span[@property="v:itemreviewed"]/text()')[0]
fType = "/".join(ele.xpath('//span[@property="v:genre"]/text()'))
director = "/".join(
ele.xpath(
'//span[@class="attrs"]/a[@rel="v:directedBy"]/text()'))
performer = "/".join(
ele.xpath(
'//span[@class="actor"]/span[@class="attrs"]/a[@rel="v:starring"]/text()'
))
score = "/".join(
ele.xpath('//strong[@property="v:average"]/text()'))
releaseDate = ",".join(
ele.xpath('//span[@property="v:initialReleaseDate"]/text()'))
timelen = "/".join(
ele.xpath('//span[@property="v:runtime"]/text()'))
introduce = ";".join(
ele.xpath('//span[@property="v:summary"]/text()'))
try:
Films = FilmsM(title, fType, director, performer, score,
releaseDate, timelen, introduce)
session.add(Films)
session.commit()
return Films.id
except InvalidRequestError as err:
print("InvalidRequestError %r" % repr(err))
session.rollback()
except Exception as e:
print("Exception %r" % repr(e))
session.rollback()
except Exception as err:
print("爬取详情Error:{0}".format(err))
def downloadImg_s(self, imgcover, id, proxies):
basedir = os.path.dirname(__file__)
hostpath = parse.urlparse(imgcover)
imgpathlist = hostpath[2].split("/")
imgname = imgpathlist[len(imgpathlist) - 1]
imgpathlist.remove(imgname)
filepath = os.path.join(basedir, '..', '..', 'static',
'/'.join(imgpathlist)[1:])
headers = {
"user-agent": userAgents[random.randrange(0, len(userAgents))]
}
if not os.path.exists(filepath):
print("路径不存在,正在创建路径~~~")
os.makedirs(filepath)
try:
res = requests.get(imgcover,
headers=headers,
proxies=proxies,
timeout=2)
with open(os.path.join(filepath, imgname), "wb") as fp:
fp.write(res.content)
fp.close()
try:
query = session.query(FilmsM).filter(
FilmsM.id == id).all()[0]
imgurl = '/static/{0}/{1}'.format(
'/'.join(imgpathlist)[1:], imgname)
query.fimgurl = imgurl
session.commit()
except InvalidRequestError as err:
print("InvalidRequestError %r" % repr(err))
session.rollback()
except Exception as e:
print("Exception %r" % repr(e))
session.rollback()
except Exception as err:
print("图片下载Error:{0}".format(err))
# ScrapyFilms('https://movie.douban.com/j/search_subjects?type=movie&tag=可播放&sort=rank&playable=on&page_limit=20&page_start={0}')
def queryPhotographyM(reqJson):
pageno = reqJson['pageno']
pagesize = reqJson['pagesize']
resJson = dict()
resJson['flag'] = "error"
resJson['msg'] = "查询失败"
resJson['data'] = []
try:
offsetnum = pagesize * (pageno - 1)
query = session.query(
PhotographyM.id, PhotographyM.title, PhotographyM.imgurlstr,
PhotographyM.desc).limit(pagesize).offset(offsetnum).all()
if len(query) != 0:
resJson['flag'] = "success"
resJson['msg'] = "查询成功"
data = []
if len(query) != 0:
for item in query:
obj = {
"id": item[0],
"title": item[1],
"imgurl": item[2],
"desc": item[3]
}
data.append(obj)
resJson['data'] = data
except InvalidRequestError:
session.rollback()
resJson['msg'] = 'InvalidRequestError %r' % repr(InvalidRequestError)
logging.error('InvalidRequestError %r' % repr(InvalidRequestError))
except Exception as err:
session.rollback()
resJson['msg'] = 'Error %r' % repr(err)
logging.error('Error %r' % repr(err))
return resJson
def detailPhotographyM(reqJson):
photoGraphyId = reqJson['photoGraphyId']
resJson = dict()
resJson['flag'] = "error"
resJson['msg'] = "查询详细失败"
resJson['data'] = []
try:
detail = session.query(PhotographyM).filter(
PhotographyM.id == photoGraphyId).all()
resJson['flag'] = "success"
resJson['msg'] = "查询详细成功"
if len(detail) != 0:
detailJson = detail[0].to_json()
resJson['data'] = detailJson
return resJson
except InvalidRequestError:
session.rollback()
resJson['msg'] = 'InvalidRequestError %r' % repr(InvalidRequestError)
logging.error('InvalidRequestError %r' % repr(InvalidRequestError))
except Exception as err:
session.rollback()
resJson['msg'] = 'Error %r' % repr(err)
logging.error('Error %r' % repr(err))
