def get_event_info(self,event_url):
if event_url:
time.sleep(random.randint(0,3))
response = requests.get(event_url,headers=random_header())
status_code = response.status_code
if status_code == 200:
event_html = response.text
eventsoup = BeautifulSoup(event_html,"html5lib")
eventid = 0
try:
eventid = int(event_url[event_url[:-1].rfind("/")+1:-1])
except:
print(event_url)
try:
activity = Activity(eventsoup, eventid)
except Exception as e:
f = open('error_url.txt', 'a')
f.write(event_url + "\r\n")
f.close()
dbhelper = DBHelper()
dbhelper.store_data(activity)
print(event_url,"数据插入完毕")
else:
f = open('error_url.txt','a')
f.write(event_url+"\r\n")
f.close()
def listen():
""""监听发送来的消息,并使用socketio向所有客户端发送消息"""
mes = {"status": "unknown error"}
text = request.form.get('data')
alias = request.form.get('alias')
file_obj = request.files.get('files')
if file_obj:
filename = file_obj.filename
file_obj.save('static/files/' + filename)
text = filename
if text:
_db = DBHelper()
_from = request.headers.get('X-Real-Ip') or request.remote_addr
_type = 'file' if file_obj else 'text'
_id = _db.insert([alias, _from, _type, text])
data = _db.select_by_id(_id)
data['alias'] = data['alias'] or data['id']
text = data['content']
data['content'] = text[:40].strip() + '...' if len(
text) >= 40 else text
socket_io.emit(data=json.dumps(data), event="mes")
mes['status'] = "success"
print(mes)
return json.dumps(mes)
def process(file_path):
"""
:param file_path: the file path which is needed to process
:return:None
"""
try:
doc_file = open(file_path, "r")
docs = doc_file.read()
doc_file.close()
except:
return
doc_len = len(docs)
d = DBHelper()
doc_id = d.insert_record_with_var(
"insert into wiki_doc(`doc_len`,`doc_path`) VALUES (%s,%s)",
(doc_len, file_path))
d = docs_to_vector(docs)
t = term()
d_t = doc_term()
for word in d:
if word not in stop_word:
term_id = 0
if t.check_term_exist(word):
term_id = t.get_term_id(word)
else:
term_id = t.insert_term(word, 0)
t.add_term_frequency(word)
d_t.insert_record(term_id, doc_id, d[word])
def CreateTask(taskName=None):
if taskName:
dbname = '%s.db' % taskName
if os.path.exists(dbname):
dbhelper = DBHelper.getInstance(dbname)
else:
dbhelper = DBHelper.getInstance(dbname)
dbhelper.execute(Enterprise.__init_table__)
def getUsers(self, titles):
db = DBHelper()
data = []
for user in db.retrieveData("SELECT * FROM `users`"):
print user[4]
userdata = self.parseXML(user[3], user[1], user[2], user[4], titles)
if userdata is not None:
data.append(userdata)
return data
def getRelatedShows(self, titlesToGet):
db = DBHelper()
inParam = ', '.join(list(map(lambda x: '%s', titlesToGet)))
sql = ("SELECT resultShow.showId, resultShow.sequel, watchedShow.showId, resultShow.sequelChain, title "
"FROM titles, shows AS resultShow, shows AS watchedShow "
"WHERE titles.showId = watchedShow.showId "
"AND title IN (%s) "
"AND resultShow.sequelChain = watchedShow.sequelChain "
"GROUP BY resultShow.showId") % inParam
return db.retrieveData(sql, tuple(titlesToGet))
def index():
"""主页面"""
# TODO 获取前10条
_db = DBHelper()
data = _db.select()
for row in data:
row['alias'] = row['alias'] or row['id']
text = row['content']
row['content'] = text[:40].strip() + '...' if len(text) >= 40 else text
return render_template("index.html", data=data)
def getShowXMLs(self, idsToGet):
db = DBHelper()
inParam = ', '.join(list(map(lambda x: '%s', idsToGet)))
sql = "SELECT showId, xml FROM shows WHERE showId IN (%s)" % inParam
xmls = db.retrieveData(sql, tuple(idsToGet))
xmlById = {}
for row in xmls:
xmlById[row[0]] = row[1]
return xmlById
def mockCheckSequel(id):
if not os.path.isfile("./xml/" + str(id) + ".xml"):
return
tree = ET.parse("./xml/" + str(id) + ".xml")
root = tree.getroot()
if root.find("relatedanime") is None:
return
for anime in root.find("relatedanime"):
if anime.attrib['type'] == "Prequel":
prequelId = anime.attrib['id']
db = DBHelper()
db.executeQuery("UPDATE shows SET sequel=" + str(id) + " WHERE showId=%s", (prequelId,))
print "Added sequel: " + str(id)
def startTask(self, taskName, progress, product=None, supplier=None):
dbname = '%s.db' % taskName
if os.path.exists(dbname):
DBHelper.getInstance(dbname)
else:
dbhelper = DBHelper.getInstance(dbname)
dbhelper.execute(Enterprise.__init_table__)
self.taskName = taskName
self.runStatus = progress
if product:
self.searchProduct(product)
else:
self.searchSupplier(supplier)
def getXML(self, id):
db = DBHelper()
lastXmlUpdate = db.retrieveData("SELECT lastXmlUpdate FROM shows WHERE showId=%s", (id,))
if len(lastXmlUpdate) < 1:
return
lastXmlUpdate = lastXmlUpdate[0][0]
if lastXmlUpdate is None or time() - lastXmlUpdate > 604800000: # 1 week
sleep(2)
url = "http://api.anidb.net:9001/httpapi?request=anime&client=seqwatcher&clientver=0&protover=1&aid=" + str(id)
r = requests.get(url)
xml = r.text.encode('utf-8')
if "anime id" not in xml: # check the response is valid
return
db.executeQuery("UPDATE shows SET xml=%s, lastXmlUpdate=%s WHERE showId=%s", (xml, time(), id))
print "Updated XML for showId: " + str(id)
def __init__(self,
lat,
lon,
min_budget,
max_budget,
min_bed,
max_bed,
min_bath,
max_bath,
distance=None,
listed_on=None,
id=None):
self.lat = lat
self.lat_rad = radians(lat)
self.lon = lon
self.lon_rad = radians(lon)
self.min_budget = min_budget
self.max_budget = max_budget
self.min_bedrooms = min_bed
self.max_bedrooms = max_bed
self.min_bathroom = min_bath
self.max_bathroom = max_bath
self.id = id
self.listed_on = listed_on
self.sql = None
self.distance = distance
self.redis = redis.StrictRedis(host='localhost', port=6379, db=0)
self.db = DBHelper.get_db()
self.score = None
def checkSequel(self, id):
# self.getXML(id)
db = DBHelper()
xml = db.retrieveData("SELECT xml FROM shows WHERE showId=%s AND xml IS NOT NULL", (id,))
if len(xml) < 1:
return
xml = xml[0][0]
root = ET.fromstring(xml)
if root.find("relatedanime") is None:
return
for anime in root.find("relatedanime"):
if anime.attrib['type'] == "Prequel":
prequelId = anime.attrib['id']
db = DBHelper()
db.executeQuery("UPDATE shows SET sequel=" + str(id) + " WHERE showId=%s", (prequelId,))
print "Added sequel: " + str(id)
def find_matching_requirements(self, distance, radius):
matches = self._get_result_from_cache()
if matches:
return matches
bounding_coordinates = GeoHelper.get_bounding_coordinates(
distance, radius, self.lat_rad, self.lon_rad)
meridian180WithinDistance = bounding_coordinates[0][
1] > bounding_coordinates[1][1]
sql = """SELECT *, {} * (acos(sin({}) * sin(lat) + cos({}) * cos(lat) * cos(lon - {}))) as distance FROM requirements WHERE (lat >= {} AND lat <= {}) AND (lon >= {} """ + (
"OR" if meridian180WithinDistance else "AND") + " lon <= {})"
params = (radius, self.lat_rad, self.lat_rad, self.lon_rad,
bounding_coordinates[0][0], bounding_coordinates[1][0],
bounding_coordinates[0][1], bounding_coordinates[1][1],
distance)
sql_where_clause = [
"min_budget*0.75 <= %s AND max_budget*1.25 >= %s " %
(self.price, self.price),
"min_bed -2<= %s AND max_bed +2>= %s" %
(self.bedroom, self.bedroom),
"min_bath -2<= %s AND max_bath +2>= %s" %
(self.bathroom, self.bathroom)
]
sql_having_clause = " HAVING distance <= {}"
sql_parts = [sql]
sql_parts.extend(sql_where_clause)
sql = " AND ".join(sql_parts) + sql_having_clause
results = DBHelper.execute_query(self.db.cursor(), sql.format(*params))
matches = []
for prop in results:
prop = Requirement(**prop)
matches.append(prop)
return MatchesCollection.order_requirements_by_score(self, matches)
def showAllForUser(self, MALname):
db = DBHelper()
root = ET.fromstring(self.getXML(MALname))
listOfShows = {}
for anime in root.findall('anime'):
title = anime.find('series_title').text.encode('utf-8')
sql = "SELECT shows.showId, shows.sequel FROM titles, shows WHERE titles.showId = shows.showId AND title = %s AND sequel IS NOT NULL"
sequel = db.retrieveData(sql, (title,))
if len(sequel) > 0:
tuple = self.parseAniDB(int(sequel[0][1]))
listOfShows[tuple[1]] = tuple
for anime in root.findall('anime'):
if anime.find('series_title').text in listOfShows.keys():
del listOfShows[anime.find('series_title').text]
return listOfShows
def parseAniDB(self, showId):
db = DBHelper()
xml = db.retrieveData("SELECT xml FROM shows WHERE showId=%s", (showId,))[0][0]
root = ET.fromstring(xml)
if root.find('startdate') is not None:
startdate = root.find('startdate').text
else:
startdate = "unknown"
if root.find('description') is not None:
desc = root.find('description').text
else:
desc = "Not available"
if root.find('titles/title') is not None:
title = root.find('titles/title').text
else:
title = "Not available"
return (startdate, title, desc, showId)
def generate_map(self):
m = folium.Map(location=[45.509484, -73.600519],
tiles="Stamen Terrain",
zoom_start=10.5)
db = DBHelper()
max_price = 300
coordinates = db.get_coordinates(max_price=max_price)
def hsv2rgb(h, s, v):
return tuple(round(i * 255) for i in colorsys.hsv_to_rgb(h, s, v))
def print_exception():
exc_type, exc_obj, tb = sys.exc_info()
f = tb.tb_frame
lineno = tb.tb_lineno
filename = f.f_code.co_filename
linecache.checkcache(filename)
line = linecache.getline(filename, lineno, f.f_globals)
print('EXCEPTION IN ({}, LINE {} "{}"): {}'.format(
filename, lineno, line.strip(), exc_obj))
print(str(sys.exc_info()))
print(f"points number: {len(coordinates)}")
# I can add marker one by one on the map
for coordinate in coordinates:
amount_n = float((max_price - coordinate['amount']) / max_price)
test_color = hsv2rgb(0.8, amount_n, 1 - amount_n)
try:
folium.Circle(
location=[coordinate['lat'], coordinate['lng']],
popup=
f"${coordinate['amount']} : https://www.airbnb.ca/rooms/{coordinate['room_id']}",
radius=10,
color='#%02x%02x%02x' % test_color,
fill=True
# fill_color='crimson'
).add_to(m)
except:
print(f"color: {test_color}")
print_exception()
# Save it as html
m.save('mymap.html')
def getTraps():
trapsArray = Constants.SUCCESS
try:
trapsArray = DBHelper.getTraps(flask_login.current_user.id)
print("getTraps:",trapsArray)
except Excetion as e:
print("GetTraps:",str(e))
trapsArray = Constants.ERROR_UNKNOWN
return flask.jsonify(trapsArray)
def notifyUsers(self, newAnime = [], user=None):
allTitles = []
db = DBHelper()
if not newAnime:
return None
for anime in newAnime:
titles = db.retrieveData("SELECT title FROM titles, shows WHERE titles.showId = shows.showId AND sequel = " + str(anime))
tmp = []
for title in titles:
tmp.append(title[0])
allTitles.append((tmp, anime,))
if user is not None:
user = db.retrieveData("SELECT * FROM `users` WHERE mal = \"" + user + "\"")
if len(user) != 1:
return None
user = user[0]
return self.parseXML(user[3], user[1], user[2], 0, titles=allTitles, first=True)
else:
return self.getUsers(allTitles)
def setTrapDetail():
status = Constants.SUCCESS
try:
req = flask.request.get_json()
trap = Constants.Trap(serial = req["serial"],
name=req["name"],
location =req["location"])
status = DBHelper.setTrapDetail(trap)
return flask.jsonify({"status":status})
except Exception as e:
status = Constants.ERROR_UNKNOWN
return flask.jsonify({"status":status})
def addNewTrap():
retVal = Constants.SUCCESS
try:
req = flask.request.get_json()
serial = req["serial"]
name = req["name"]
location = req["location"]
email = flask_login.current_user.id
print(serial,name,location,email)
retVal = DBHelper.addNewTrap(email,serial,name,location)
except Exception as e:
print("Exception: main: addNewTrap:"+str(e))
retVal = Constants.ERROR_UNKNOWN
return flask.jsonify({"status":retVal})
def getTrapDetails():
try:
req = flask.request.get_json()
serial = req["serial"]
trap = DBHelper.getTrapDetails(serial)
if trap == None:
raise Exception
print("main: getTrapDetails: trap:",str(trap))
return flask.jsonify({"status":Constants.SUCCESS,
"name":trap.name,
"location":trap.location,
"ap":trap.ap})
except Exception as e:
print("Exception: main: getTrapDetails:",str(e))
return flask.jsonify({"status":Constants.ERROR_UNKNOWN})
def sigup():
try:
data = flask.request.get_json()
serial = data["serial"]
email = data["email"]
password = data["password"]
retVal = DBHelper.createAccount(serial,email,password)
if retVal!=Constants.SUCCESS:
return flask.jsonify({"status":retVal})
os.mkdir(trapDataDirPath+"/"+serial)
return flask.jsonify({"status":retVal})
except Exception as e:
print("Exception:"+str(e))
return flask.jsonify({"status":Constants.ERROR_UNKNOWN})
def login():
try:
if flask.request.method == 'POST':
data = flask.request.get_json()
email = data["email"].replace("\'","").replace("\"","")
password = data["password"].replace("\'","").replace("\"","")
retVal = DBHelper.checkCredential(email,password)
# if email and password true
if retVal == Constants.SUCCESS:
user = User(email)
flask_login.login_user(user)
print("login success")
return flask.jsonify({"status":Constants.SUCCESS})
elif retVal == Constants.ERROR_WRONG_EMAIL_PASS: # invalid credentials
return flask.jsonify({"status":Constants.ERROR_WRONG_EMAIL_PASS})
else: # unknwon server error
return flask.jsonify({"status":Constants.ERROR_UNKNOWN})
except Exception as e:
print("Exception:",str(e))
return flask.render_template('login.html')
def addNewShows(self):
db = DBHelper()
lastId = db.retrieveData("SELECT * FROM shows ORDER BY showId DESC LIMIT 1", ())
if len(lastId) > 0:
lastId = lastId[0][0]
else:
lastId = 0
tree = ET.parse('./allshows.xml')
root = tree.getroot()
for child in root:
id = int(child.attrib['aid'])
if id > lastId:
db.executeQuery("INSERT INTO shows (showId) VALUES (" + str(id) + ")", ())
for title in child:
lang = title.attrib['{http://www.w3.org/XML/1998/namespace}lang']
if (lang == "en") or (lang == "x-jat"):
db.executeQuery("INSERT INTO titles (showId, title) VALUES (" + str(id) + ", %s)", (title.text,))
def __init__(self,
lat,
lon,
price,
bed,
bath,
listed_on=None,
distance=None,
id=None):
self.lat = lat
self.lat_rad = radians(lat)
self.lon = lon
self.lon_rad = radians(lon)
self.bedroom = bed
self.bathroom = bath
self.price = price
self.score = 0
self.sql = None
self.distance = distance
self.listed_on = listed_on
self.id = id
self.redis = redis.StrictRedis(host='localhost', port=6379, db=0)
self.db = DBHelper.get_db()
def find_matching_properties(self, distance, radius):
# matches = self._get_result_from_cache()
# if matches:
# return matches
bounding_coordinates = GeoHelper.get_bounding_coordinates(
distance, radius, self.lat_rad, self.lon_rad)
meridian180WithinDistance = bounding_coordinates[0][
1] > bounding_coordinates[1][1]
sql = """SELECT *, {} * (acos(sin({}) * sin(lat) + cos({}) * cos(lat) * cos(lon - {}))) as distance FROM properties_new WHERE (lat >= {} AND lat <= {}) AND (lon >= {} """ + (
"OR" if meridian180WithinDistance else "AND") + " lon <= {})"
params = (radius, self.lat_rad, self.lat_rad, self.lon_rad,
bounding_coordinates[0][0], bounding_coordinates[1][0],
bounding_coordinates[0][1], bounding_coordinates[1][1],
distance)
sql_where_clause = [
"price BETWEEN %s AND %s" %
((self.min_budget * 0.75 or self.max_budget * 0.75),
(self.max_budget * 1.25 or self.min_budget * 1.25)),
"bed BETWEEN %s AND %s" %
((self.min_bedrooms - 2 or self.max_bedrooms - 2),
(self.max_bedrooms + 2 or self.min_bedrooms + 2)),
"bath BETWEEN %s AND %s" %
((self.min_bathroom - 2 or self.max_bathroom - 2),
(self.max_bathroom + 2 or self.min_bathroom + 2))
]
sql_having_clause = " HAVING distance <= {}"
sql_parts = [sql]
sql_parts.extend(sql_where_clause)
sql = " AND ".join(sql_parts) + sql_having_clause
sql = sql.format(*params)
results = DBHelper.execute_query(self.db.cursor(), sql)
matches = []
for prop in results:
prop = Properties(**prop)
matches.append(prop)
return MatchesCollection.order_matches_by_score(self, matches)
def operation():
"""operation"""
if request.method == 'GET':
_id = request.args.get('id')
if not _id:
return {'msg': 'error'}
_db = DBHelper()
data = _db.select_by_id(_id)
filename = data['content']
response = make_response(
send_from_directory('static/files', filename, as_attachment=True))
response.headers[
"Content-Disposition"] = "attachment; filename={}".format(
filename.encode().decode('latin-1'))
return response
elif request.method == 'POST':
_id = request.form.get('id')
if not _id:
return {'msg': 'error'}
_db = DBHelper()
data = _db.select_by_id(_id)
return json.dumps({'text': data['content']})
def main():
app = Application()
DBHelper().genschema()
app.listen(os.getenv("PORT", 3000))
IOLoop.instance().start()
# import Session17E
# import Session17E as ss
# from Session17E import hello, bye
from practice17m import *
# import db.DBHelper
# import db.DBHelper as db
from db import DBHelper
"""
Session17E.hello("John")
Session17E.bye("John")
print(Session17E.companyName)
eRef = Session17E.Employee(101, "Fionna", 30000)
eRef.showEmployee()
"""
"""
ss.hello("John")
ss.bye("John")
print(ss.companyName)
eRef = ss.Employee(101, "Fionna", 30000)
eRef.showEmployee()
"""
# hello("John")
# bye("John")
# db.saveCustomer("john","+919876552512", "*****@*****.**")
DBHelper.saveCustomer("john", "+919876552512", "*****@*****.**")
def __init__(self):
self.db = DBHelper()
ssl._create_default_https_context = ssl._create_unverified_context
print('爬虫开始行动了……')
class QiuBaiSpider():
def __init__(self):
self.db = DBHelper()
ssl._create_default_https_context = ssl._create_unverified_context
print('爬虫开始行动了……')
def __del__(self):
print('感谢有你，我要走了……')
self.db.close()
def request(self, url):
# 创建opener浏览器对象,并且设置代理处理器
opener = request.build_opener(request.ProxyHandler(proxies={'http': random.choice(settings.proxies['http'])}))
req = request.Request(url, headers=settings.headers)
resp = opener.open(req)
if resp.status == 200:
print('ok')
html = resp.read().decode()
# print(html)
return html
def parse(self, html):
# 创建一个et对象
et = etree.HTML(html)
# print(et)
authors = et.xpath(settings.author_path)
# print(authors)
for author in authors: # author 的类型为 <class 'lxml.etree._Element'>
# print(author)
# print(type(author))
try:
# try不存在作用域
home = author.xpath(settings.home_path)[0] # './a/@href'
id = home.split('/')[-2] # /users/38248088/
name = author.xpath(settings.name_path)[0]
age = author.xpath(settings.age_path)[0]
img = 'http:' + author.xpath(settings.src_path)[0].split('?')[0]
# print(home)
except:
pass
else:
item = UserItem(id, name, age, img, home)
# print(item)
# 将数据存放到数据库
self.db.save(item)
self.saveImg(img, id)
# 读取下一页的链接
try:
next_url = settings.start_url + et.xpath(settings.next_page_path)[0]
print(next_url)
except:
pass
else:
return next_url
def saveImg(self, url, id):
filename = './head/{}.{}'.format(id, url.split('.')[-1])
if os.path.exists(filename):
return
request.urlretrieve(url, filename=filename)
print(filename, '图片下载成功！！！')
def run(self):
next_url = settings.start_url
while True:
html = self.request(next_url)
# 解析网页并获取下一次请求的路径
next_url = self.parse(html)
if not next_url:
break
# -*-coding:utf-8-*-
from db import DBHelper
from item import UserItem
# 测试案例
db_ = DBHelper()
print(db_.exist('user', 1))
item = UserItem(1, 'lili', '14', 'xiha', 'http://')
db_.save()
import os
import time
from threading import Thread
from db import DBHelper
# import csv
db = DBHelper('localhost', 'webasudd', 'asudduser', 'asuddp@ss')
class MyThread(Thread):
"""
A threading example
"""
def __init__(self, node):
"""Инициализация потока"""
Thread.__init__(self)
self.node = node
def run(self):
"""Запуск потока"""
ping_list = []
if len(self.node[5]) > 0:
ping_list.append(os.system(f"ping -c 3 {self.node[5]} > /dev/null"))
if len(self.node[6]) > 0:
ping_list.append(os.system(f"ping -c 3 {self.node[6]} > /dev/null"))
if len(self.node[7]) > 0:
ping_list.append(os.system(f"ping -c 3 {self.node[7]} > /dev/null"))
if sum(ping_list) > 0:
def setDb(self, taskdb):
if not os.path.exists(taskdb):
self.taskDb = DBHelper.getInstance(taskdb)
self.taskDb.execute(Task.__init_table__)
else:
self.taskDb = DBHelper.getInstance(taskdb)
import string
from db import DBHelper
db = DBHelper.DBHelper("10.16.6.120", "fanxing", "kugou2014", "d_fanxing")
def select(sql):
return db.select(sql)
def insert(p_table_name, p_data):
key = string.join(p_data.keys(), "`,`")
values = [str(d) for d in p_data.values()]
value = string.join(values, "','")
real_sql = "INSERT INTO " + p_table_name + " (`" + key + "`) VALUES ('" + value + "')"
print(real_sql)
return db.insert(real_sql)
def insertBySql(sql):
return db.insert(sql)
import folium
from db import DBHelper
import colorsys
import sys
import linecache
import matplotlib.pyplot as plt
import matplotlib
font = {'size': 36}
matplotlib.rc('font', **font)
db = DBHelper()
class MapGenerator:
def generate_map(self):
m = folium.Map(location=[45.509484, -73.600519],
tiles="Stamen Terrain",
zoom_start=10.5)
db = DBHelper()
max_price = 300
coordinates = db.get_coordinates(max_price=max_price)
def hsv2rgb(h, s, v):
return tuple(round(i * 255) for i in colorsys.hsv_to_rgb(h, s, v))
def print_exception():
exc_type, exc_obj, tb = sys.exc_info()
f = tb.tb_frame
lineno = tb.tb_lineno
