def get_event_info(self,event_url): if event_url: time.sleep(random.randint(0,3)) response = requests.get(event_url,headers=random_header()) status_code = response.status_code if status_code == 200: event_html = response.text eventsoup = BeautifulSoup(event_html,"html5lib") eventid = 0 try: eventid = int(event_url[event_url[:-1].rfind("/")+1:-1]) except: print(event_url) try: activity = Activity(eventsoup, eventid) except Exception as e: f = open('error_url.txt', 'a') f.write(event_url + "\r\n") f.close() dbhelper = DBHelper() dbhelper.store_data(activity) print(event_url,"数据插入完毕") else: f = open('error_url.txt','a') f.write(event_url+"\r\n") f.close()
def listen(): """"监听发送来的消息,并使用socketio向所有客户端发送消息""" mes = {"status": "unknown error"} text = request.form.get('data') alias = request.form.get('alias') file_obj = request.files.get('files') if file_obj: filename = file_obj.filename file_obj.save('static/files/' + filename) text = filename if text: _db = DBHelper() _from = request.headers.get('X-Real-Ip') or request.remote_addr _type = 'file' if file_obj else 'text' _id = _db.insert([alias, _from, _type, text]) data = _db.select_by_id(_id) data['alias'] = data['alias'] or data['id'] text = data['content'] data['content'] = text[:40].strip() + '...' if len( text) >= 40 else text socket_io.emit(data=json.dumps(data), event="mes") mes['status'] = "success" print(mes) return json.dumps(mes)
def process(file_path): """ :param file_path: the file path which is needed to process :return:None """ try: doc_file = open(file_path, "r") docs = doc_file.read() doc_file.close() except: return doc_len = len(docs) d = DBHelper() doc_id = d.insert_record_with_var( "insert into wiki_doc(`doc_len`,`doc_path`) VALUES (%s,%s)", (doc_len, file_path)) d = docs_to_vector(docs) t = term() d_t = doc_term() for word in d: if word not in stop_word: term_id = 0 if t.check_term_exist(word): term_id = t.get_term_id(word) else: term_id = t.insert_term(word, 0) t.add_term_frequency(word) d_t.insert_record(term_id, doc_id, d[word])
def CreateTask(taskName=None): if taskName: dbname = '%s.db' % taskName if os.path.exists(dbname): dbhelper = DBHelper.getInstance(dbname) else: dbhelper = DBHelper.getInstance(dbname) dbhelper.execute(Enterprise.__init_table__)
def getUsers(self, titles): db = DBHelper() data = [] for user in db.retrieveData("SELECT * FROM `users`"): print user[4] userdata = self.parseXML(user[3], user[1], user[2], user[4], titles) if userdata is not None: data.append(userdata) return data
def getRelatedShows(self, titlesToGet): db = DBHelper() inParam = ', '.join(list(map(lambda x: '%s', titlesToGet))) sql = ("SELECT resultShow.showId, resultShow.sequel, watchedShow.showId, resultShow.sequelChain, title " "FROM titles, shows AS resultShow, shows AS watchedShow " "WHERE titles.showId = watchedShow.showId " "AND title IN (%s) " "AND resultShow.sequelChain = watchedShow.sequelChain " "GROUP BY resultShow.showId") % inParam return db.retrieveData(sql, tuple(titlesToGet))
def index(): """主页面""" # TODO 获取前10条 _db = DBHelper() data = _db.select() for row in data: row['alias'] = row['alias'] or row['id'] text = row['content'] row['content'] = text[:40].strip() + '...' if len(text) >= 40 else text return render_template("index.html", data=data)
def getShowXMLs(self, idsToGet): db = DBHelper() inParam = ', '.join(list(map(lambda x: '%s', idsToGet))) sql = "SELECT showId, xml FROM shows WHERE showId IN (%s)" % inParam xmls = db.retrieveData(sql, tuple(idsToGet)) xmlById = {} for row in xmls: xmlById[row[0]] = row[1] return xmlById
def mockCheckSequel(id): if not os.path.isfile("./xml/" + str(id) + ".xml"): return tree = ET.parse("./xml/" + str(id) + ".xml") root = tree.getroot() if root.find("relatedanime") is None: return for anime in root.find("relatedanime"): if anime.attrib['type'] == "Prequel": prequelId = anime.attrib['id'] db = DBHelper() db.executeQuery("UPDATE shows SET sequel=" + str(id) + " WHERE showId=%s", (prequelId,)) print "Added sequel: " + str(id)
def startTask(self, taskName, progress, product=None, supplier=None): dbname = '%s.db' % taskName if os.path.exists(dbname): DBHelper.getInstance(dbname) else: dbhelper = DBHelper.getInstance(dbname) dbhelper.execute(Enterprise.__init_table__) self.taskName = taskName self.runStatus = progress if product: self.searchProduct(product) else: self.searchSupplier(supplier)
def getXML(self, id): db = DBHelper() lastXmlUpdate = db.retrieveData("SELECT lastXmlUpdate FROM shows WHERE showId=%s", (id,)) if len(lastXmlUpdate) < 1: return lastXmlUpdate = lastXmlUpdate[0][0] if lastXmlUpdate is None or time() - lastXmlUpdate > 604800000: # 1 week sleep(2) url = "http://api.anidb.net:9001/httpapi?request=anime&client=seqwatcher&clientver=0&protover=1&aid=" + str(id) r = requests.get(url) xml = r.text.encode('utf-8') if "anime id" not in xml: # check the response is valid return db.executeQuery("UPDATE shows SET xml=%s, lastXmlUpdate=%s WHERE showId=%s", (xml, time(), id)) print "Updated XML for showId: " + str(id)
def __init__(self, lat, lon, min_budget, max_budget, min_bed, max_bed, min_bath, max_bath, distance=None, listed_on=None, id=None): self.lat = lat self.lat_rad = radians(lat) self.lon = lon self.lon_rad = radians(lon) self.min_budget = min_budget self.max_budget = max_budget self.min_bedrooms = min_bed self.max_bedrooms = max_bed self.min_bathroom = min_bath self.max_bathroom = max_bath self.id = id self.listed_on = listed_on self.sql = None self.distance = distance self.redis = redis.StrictRedis(host='localhost', port=6379, db=0) self.db = DBHelper.get_db() self.score = None
def checkSequel(self, id): # self.getXML(id) db = DBHelper() xml = db.retrieveData("SELECT xml FROM shows WHERE showId=%s AND xml IS NOT NULL", (id,)) if len(xml) < 1: return xml = xml[0][0] root = ET.fromstring(xml) if root.find("relatedanime") is None: return for anime in root.find("relatedanime"): if anime.attrib['type'] == "Prequel": prequelId = anime.attrib['id'] db = DBHelper() db.executeQuery("UPDATE shows SET sequel=" + str(id) + " WHERE showId=%s", (prequelId,)) print "Added sequel: " + str(id)
def find_matching_requirements(self, distance, radius): matches = self._get_result_from_cache() if matches: return matches bounding_coordinates = GeoHelper.get_bounding_coordinates( distance, radius, self.lat_rad, self.lon_rad) meridian180WithinDistance = bounding_coordinates[0][ 1] > bounding_coordinates[1][1] sql = """SELECT *, {} * (acos(sin({}) * sin(lat) + cos({}) * cos(lat) * cos(lon - {}))) as distance FROM requirements WHERE (lat >= {} AND lat <= {}) AND (lon >= {} """ + ( "OR" if meridian180WithinDistance else "AND") + " lon <= {})" params = (radius, self.lat_rad, self.lat_rad, self.lon_rad, bounding_coordinates[0][0], bounding_coordinates[1][0], bounding_coordinates[0][1], bounding_coordinates[1][1], distance) sql_where_clause = [ "min_budget*0.75 <= %s AND max_budget*1.25 >= %s " % (self.price, self.price), "min_bed -2<= %s AND max_bed +2>= %s" % (self.bedroom, self.bedroom), "min_bath -2<= %s AND max_bath +2>= %s" % (self.bathroom, self.bathroom) ] sql_having_clause = " HAVING distance <= {}" sql_parts = [sql] sql_parts.extend(sql_where_clause) sql = " AND ".join(sql_parts) + sql_having_clause results = DBHelper.execute_query(self.db.cursor(), sql.format(*params)) matches = [] for prop in results: prop = Requirement(**prop) matches.append(prop) return MatchesCollection.order_requirements_by_score(self, matches)
def showAllForUser(self, MALname): db = DBHelper() root = ET.fromstring(self.getXML(MALname)) listOfShows = {} for anime in root.findall('anime'): title = anime.find('series_title').text.encode('utf-8') sql = "SELECT shows.showId, shows.sequel FROM titles, shows WHERE titles.showId = shows.showId AND title = %s AND sequel IS NOT NULL" sequel = db.retrieveData(sql, (title,)) if len(sequel) > 0: tuple = self.parseAniDB(int(sequel[0][1])) listOfShows[tuple[1]] = tuple for anime in root.findall('anime'): if anime.find('series_title').text in listOfShows.keys(): del listOfShows[anime.find('series_title').text] return listOfShows
def parseAniDB(self, showId): db = DBHelper() xml = db.retrieveData("SELECT xml FROM shows WHERE showId=%s", (showId,))[0][0] root = ET.fromstring(xml) if root.find('startdate') is not None: startdate = root.find('startdate').text else: startdate = "unknown" if root.find('description') is not None: desc = root.find('description').text else: desc = "Not available" if root.find('titles/title') is not None: title = root.find('titles/title').text else: title = "Not available" return (startdate, title, desc, showId)
def generate_map(self): m = folium.Map(location=[45.509484, -73.600519], tiles="Stamen Terrain", zoom_start=10.5) db = DBHelper() max_price = 300 coordinates = db.get_coordinates(max_price=max_price) def hsv2rgb(h, s, v): return tuple(round(i * 255) for i in colorsys.hsv_to_rgb(h, s, v)) def print_exception(): exc_type, exc_obj, tb = sys.exc_info() f = tb.tb_frame lineno = tb.tb_lineno filename = f.f_code.co_filename linecache.checkcache(filename) line = linecache.getline(filename, lineno, f.f_globals) print('EXCEPTION IN ({}, LINE {} "{}"): {}'.format( filename, lineno, line.strip(), exc_obj)) print(str(sys.exc_info())) print(f"points number: {len(coordinates)}") # I can add marker one by one on the map for coordinate in coordinates: amount_n = float((max_price - coordinate['amount']) / max_price) test_color = hsv2rgb(0.8, amount_n, 1 - amount_n) try: folium.Circle( location=[coordinate['lat'], coordinate['lng']], popup= f"${coordinate['amount']} : https://www.airbnb.ca/rooms/{coordinate['room_id']}", radius=10, color='#%02x%02x%02x' % test_color, fill=True # fill_color='crimson' ).add_to(m) except: print(f"color: {test_color}") print_exception() # Save it as html m.save('mymap.html')
def getTraps(): trapsArray = Constants.SUCCESS try: trapsArray = DBHelper.getTraps(flask_login.current_user.id) print("getTraps:",trapsArray) except Excetion as e: print("GetTraps:",str(e)) trapsArray = Constants.ERROR_UNKNOWN return flask.jsonify(trapsArray)
def notifyUsers(self, newAnime = [], user=None): allTitles = [] db = DBHelper() if not newAnime: return None for anime in newAnime: titles = db.retrieveData("SELECT title FROM titles, shows WHERE titles.showId = shows.showId AND sequel = " + str(anime)) tmp = [] for title in titles: tmp.append(title[0]) allTitles.append((tmp, anime,)) if user is not None: user = db.retrieveData("SELECT * FROM `users` WHERE mal = \"" + user + "\"") if len(user) != 1: return None user = user[0] return self.parseXML(user[3], user[1], user[2], 0, titles=allTitles, first=True) else: return self.getUsers(allTitles)
def setTrapDetail(): status = Constants.SUCCESS try: req = flask.request.get_json() trap = Constants.Trap(serial = req["serial"], name=req["name"], location =req["location"]) status = DBHelper.setTrapDetail(trap) return flask.jsonify({"status":status}) except Exception as e: status = Constants.ERROR_UNKNOWN return flask.jsonify({"status":status})
def addNewTrap(): retVal = Constants.SUCCESS try: req = flask.request.get_json() serial = req["serial"] name = req["name"] location = req["location"] email = flask_login.current_user.id print(serial,name,location,email) retVal = DBHelper.addNewTrap(email,serial,name,location) except Exception as e: print("Exception: main: addNewTrap:"+str(e)) retVal = Constants.ERROR_UNKNOWN return flask.jsonify({"status":retVal})
def getTrapDetails(): try: req = flask.request.get_json() serial = req["serial"] trap = DBHelper.getTrapDetails(serial) if trap == None: raise Exception print("main: getTrapDetails: trap:",str(trap)) return flask.jsonify({"status":Constants.SUCCESS, "name":trap.name, "location":trap.location, "ap":trap.ap}) except Exception as e: print("Exception: main: getTrapDetails:",str(e)) return flask.jsonify({"status":Constants.ERROR_UNKNOWN})
def sigup(): try: data = flask.request.get_json() serial = data["serial"] email = data["email"] password = data["password"] retVal = DBHelper.createAccount(serial,email,password) if retVal!=Constants.SUCCESS: return flask.jsonify({"status":retVal}) os.mkdir(trapDataDirPath+"/"+serial) return flask.jsonify({"status":retVal}) except Exception as e: print("Exception:"+str(e)) return flask.jsonify({"status":Constants.ERROR_UNKNOWN})
def login(): try: if flask.request.method == 'POST': data = flask.request.get_json() email = data["email"].replace("\'","").replace("\"","") password = data["password"].replace("\'","").replace("\"","") retVal = DBHelper.checkCredential(email,password) # if email and password true if retVal == Constants.SUCCESS: user = User(email) flask_login.login_user(user) print("login success") return flask.jsonify({"status":Constants.SUCCESS}) elif retVal == Constants.ERROR_WRONG_EMAIL_PASS: # invalid credentials return flask.jsonify({"status":Constants.ERROR_WRONG_EMAIL_PASS}) else: # unknwon server error return flask.jsonify({"status":Constants.ERROR_UNKNOWN}) except Exception as e: print("Exception:",str(e)) return flask.render_template('login.html')
def addNewShows(self): db = DBHelper() lastId = db.retrieveData("SELECT * FROM shows ORDER BY showId DESC LIMIT 1", ()) if len(lastId) > 0: lastId = lastId[0][0] else: lastId = 0 tree = ET.parse('./allshows.xml') root = tree.getroot() for child in root: id = int(child.attrib['aid']) if id > lastId: db.executeQuery("INSERT INTO shows (showId) VALUES (" + str(id) + ")", ()) for title in child: lang = title.attrib['{http://www.w3.org/XML/1998/namespace}lang'] if (lang == "en") or (lang == "x-jat"): db.executeQuery("INSERT INTO titles (showId, title) VALUES (" + str(id) + ", %s)", (title.text,))
def __init__(self, lat, lon, price, bed, bath, listed_on=None, distance=None, id=None): self.lat = lat self.lat_rad = radians(lat) self.lon = lon self.lon_rad = radians(lon) self.bedroom = bed self.bathroom = bath self.price = price self.score = 0 self.sql = None self.distance = distance self.listed_on = listed_on self.id = id self.redis = redis.StrictRedis(host='localhost', port=6379, db=0) self.db = DBHelper.get_db()
def find_matching_properties(self, distance, radius): # matches = self._get_result_from_cache() # if matches: # return matches bounding_coordinates = GeoHelper.get_bounding_coordinates( distance, radius, self.lat_rad, self.lon_rad) meridian180WithinDistance = bounding_coordinates[0][ 1] > bounding_coordinates[1][1] sql = """SELECT *, {} * (acos(sin({}) * sin(lat) + cos({}) * cos(lat) * cos(lon - {}))) as distance FROM properties_new WHERE (lat >= {} AND lat <= {}) AND (lon >= {} """ + ( "OR" if meridian180WithinDistance else "AND") + " lon <= {})" params = (radius, self.lat_rad, self.lat_rad, self.lon_rad, bounding_coordinates[0][0], bounding_coordinates[1][0], bounding_coordinates[0][1], bounding_coordinates[1][1], distance) sql_where_clause = [ "price BETWEEN %s AND %s" % ((self.min_budget * 0.75 or self.max_budget * 0.75), (self.max_budget * 1.25 or self.min_budget * 1.25)), "bed BETWEEN %s AND %s" % ((self.min_bedrooms - 2 or self.max_bedrooms - 2), (self.max_bedrooms + 2 or self.min_bedrooms + 2)), "bath BETWEEN %s AND %s" % ((self.min_bathroom - 2 or self.max_bathroom - 2), (self.max_bathroom + 2 or self.min_bathroom + 2)) ] sql_having_clause = " HAVING distance <= {}" sql_parts = [sql] sql_parts.extend(sql_where_clause) sql = " AND ".join(sql_parts) + sql_having_clause sql = sql.format(*params) results = DBHelper.execute_query(self.db.cursor(), sql) matches = [] for prop in results: prop = Properties(**prop) matches.append(prop) return MatchesCollection.order_matches_by_score(self, matches)
def operation(): """operation""" if request.method == 'GET': _id = request.args.get('id') if not _id: return {'msg': 'error'} _db = DBHelper() data = _db.select_by_id(_id) filename = data['content'] response = make_response( send_from_directory('static/files', filename, as_attachment=True)) response.headers[ "Content-Disposition"] = "attachment; filename={}".format( filename.encode().decode('latin-1')) return response elif request.method == 'POST': _id = request.form.get('id') if not _id: return {'msg': 'error'} _db = DBHelper() data = _db.select_by_id(_id) return json.dumps({'text': data['content']})
def main(): app = Application() DBHelper().genschema() app.listen(os.getenv("PORT", 3000)) IOLoop.instance().start()
# import Session17E # import Session17E as ss # from Session17E import hello, bye from practice17m import * # import db.DBHelper # import db.DBHelper as db from db import DBHelper """ Session17E.hello("John") Session17E.bye("John") print(Session17E.companyName) eRef = Session17E.Employee(101, "Fionna", 30000) eRef.showEmployee() """ """ ss.hello("John") ss.bye("John") print(ss.companyName) eRef = ss.Employee(101, "Fionna", 30000) eRef.showEmployee() """ # hello("John") # bye("John") # db.saveCustomer("john","+919876552512", "*****@*****.**") DBHelper.saveCustomer("john", "+919876552512", "*****@*****.**")
def __init__(self): self.db = DBHelper() ssl._create_default_https_context = ssl._create_unverified_context print('爬虫开始行动了……')
class QiuBaiSpider(): def __init__(self): self.db = DBHelper() ssl._create_default_https_context = ssl._create_unverified_context print('爬虫开始行动了……') def __del__(self): print('感谢有你,我要走了……') self.db.close() def request(self, url): # 创建opener浏览器对象,并且设置代理处理器 opener = request.build_opener(request.ProxyHandler(proxies={'http': random.choice(settings.proxies['http'])})) req = request.Request(url, headers=settings.headers) resp = opener.open(req) if resp.status == 200: print('ok') html = resp.read().decode() # print(html) return html def parse(self, html): # 创建一个et对象 et = etree.HTML(html) # print(et) authors = et.xpath(settings.author_path) # print(authors) for author in authors: # author 的类型为 <class 'lxml.etree._Element'> # print(author) # print(type(author)) try: # try不存在作用域 home = author.xpath(settings.home_path)[0] # './a/@href' id = home.split('/')[-2] # /users/38248088/ name = author.xpath(settings.name_path)[0] age = author.xpath(settings.age_path)[0] img = 'http:' + author.xpath(settings.src_path)[0].split('?')[0] # print(home) except: pass else: item = UserItem(id, name, age, img, home) # print(item) # 将数据存放到数据库 self.db.save(item) self.saveImg(img, id) # 读取下一页的链接 try: next_url = settings.start_url + et.xpath(settings.next_page_path)[0] print(next_url) except: pass else: return next_url def saveImg(self, url, id): filename = './head/{}.{}'.format(id, url.split('.')[-1]) if os.path.exists(filename): return request.urlretrieve(url, filename=filename) print(filename, '图片下载成功!!!') def run(self): next_url = settings.start_url while True: html = self.request(next_url) # 解析网页并获取下一次请求的路径 next_url = self.parse(html) if not next_url: break
# -*-coding:utf-8-*- from db import DBHelper from item import UserItem # 测试案例 db_ = DBHelper() print(db_.exist('user', 1)) item = UserItem(1, 'lili', '14', 'xiha', 'http://') db_.save()
import os import time from threading import Thread from db import DBHelper # import csv db = DBHelper('localhost', 'webasudd', 'asudduser', 'asuddp@ss') class MyThread(Thread): """ A threading example """ def __init__(self, node): """Инициализация потока""" Thread.__init__(self) self.node = node def run(self): """Запуск потока""" ping_list = [] if len(self.node[5]) > 0: ping_list.append(os.system(f"ping -c 3 {self.node[5]} > /dev/null")) if len(self.node[6]) > 0: ping_list.append(os.system(f"ping -c 3 {self.node[6]} > /dev/null")) if len(self.node[7]) > 0: ping_list.append(os.system(f"ping -c 3 {self.node[7]} > /dev/null")) if sum(ping_list) > 0:
def setDb(self, taskdb): if not os.path.exists(taskdb): self.taskDb = DBHelper.getInstance(taskdb) self.taskDb.execute(Task.__init_table__) else: self.taskDb = DBHelper.getInstance(taskdb)
import string from db import DBHelper db = DBHelper.DBHelper("10.16.6.120", "fanxing", "kugou2014", "d_fanxing") def select(sql): return db.select(sql) def insert(p_table_name, p_data): key = string.join(p_data.keys(), "`,`") values = [str(d) for d in p_data.values()] value = string.join(values, "','") real_sql = "INSERT INTO " + p_table_name + " (`" + key + "`) VALUES ('" + value + "')" print(real_sql) return db.insert(real_sql) def insertBySql(sql): return db.insert(sql)
import folium from db import DBHelper import colorsys import sys import linecache import matplotlib.pyplot as plt import matplotlib font = {'size': 36} matplotlib.rc('font', **font) db = DBHelper() class MapGenerator: def generate_map(self): m = folium.Map(location=[45.509484, -73.600519], tiles="Stamen Terrain", zoom_start=10.5) db = DBHelper() max_price = 300 coordinates = db.get_coordinates(max_price=max_price) def hsv2rgb(h, s, v): return tuple(round(i * 255) for i in colorsys.hsv_to_rgb(h, s, v)) def print_exception(): exc_type, exc_obj, tb = sys.exc_info() f = tb.tb_frame lineno = tb.tb_lineno