def __init__(self, squeue): super(SilentProcess, self).__init__() multiprocessing.Process.__init__(self) self.daemon = True self.squeue = squeue self.destFile = os.path.join(UKSC_CACHE_DIR,"uksc.db") self.connect = sqlite3.connect(self.destFile, timeout=30.0, check_same_thread=False) self.connect.execute('pragma journal_mode=wal;') self.cursor = self.connect.cursor() self.premoter = PistonRemoter(service_root=UBUNTUKYLIN_SERVER)
def __init__(self): self.updatecount = 0 srcFile = os.path.join(UBUNTUKYLIN_DATA_PATH, "uksc.db") destFile = os.path.join(UKSC_CACHE_DIR, "uksc.db") # no cache file, copy if not os.path.exists(destFile): if not os.path.exists(srcFile): if (Globals.DEBUG_SWITCH): print("error with db file") return open(destFile, "wb").write(open(srcFile, "rb").read()) self.connect = sqlite3.connect(destFile, timeout=30.0, check_same_thread=False) self.connect.execute('pragma journal_mode=wal;') self.connect.cursor() self.cursor = self.connect.cursor() self.cat_list = [] # cache file need update, copy if self.is_cachedb_need_update(): open(destFile, "wb").write(open(srcFile, "rb").read()) # piston remoter to ukscs self.premoter = PistonRemoter(service_root=UBUNTUKYLIN_SERVER) #___________________________add by zhangxin for xapiandb update___________________________# xapian_srcFile = XAPIAN_DB_SOURCE_PATH xapian_destFile = os.path.join(UKSC_CACHE_DIR, "xapiandb") # no cache file, copy if not os.path.exists(xapian_destFile): if not os.path.exists(xapian_srcFile): if (Globals.DEBUG_SWITCH): print( "No xapiandb source in /usr/share/ubuntu-kylin-software-center/data/,please reinstall it" ) return copytree(xapian_srcFile, xapian_destFile) if (Globals.DEBUG_SWITCH): print("Xapiandb has been copy to cache") # cache xapiandb need update, copy if self.is_xapiancachedb_need_update(): rmtree(xapian_destFile) copytree(xapian_srcFile, xapian_destFile) if (Globals.DEBUG_SWITCH): print("cache xapiandb versin updated")
def __init__(self, squeue): super(SilentProcess, self).__init__() multiprocessing.Process.__init__(self) self.daemon = True self.squeue = squeue self.destFile = os.path.join(UKSC_CACHE_DIR,"uksc.db") self.connect = sqlite3.connect(self.destFile, check_same_thread=False) self.cursor = self.connect.cursor() self.premoter = PistonRemoter(service_root=UBUNTUKYLIN_SERVER)
def __init__(self): self.updatecount = 0 srcFile = os.path.join(UBUNTUKYLIN_DATA_PATH,"uksc.db") destFile = os.path.join(UKSC_CACHE_DIR,"uksc.db") # no cache file, copy if not os.path.exists(destFile): if not os.path.exists(srcFile): print "error with db file" return open(destFile, "wb").write(open(srcFile, "rb").read()) self.connect = sqlite3.connect(destFile, check_same_thread=False) self.cursor = self.connect.cursor() self.cat_list = [] # cache file need update, copy if self.is_cachedb_need_update(): open(destFile, "wb").write(open(srcFile, "rb").read()) # piston remoter to ukscs self.premoter = PistonRemoter(service_root=UBUNTUKYLIN_SERVER)
def __init__(self, parent=None): QWidget.__init__(self, parent) self.ui_init() self.setWindowFlags(Qt.FramelessWindowHint) self.ui.bg.lower() self.move(280, 60) self.ui.topWidget.raise_() palette = QPalette() brush = QBrush(QColor(0, 0, 0, 0)) brush.setStyle(Qt.SolidPattern) palette.setBrush(QPalette.All, QPalette.Base, brush) self.premoter = PistonRemoter() #self.ui.btnAdd.setFocusPolicy(Qt.NoFocus) self.ui.groupBox.setFocusPolicy(Qt.NoFocus) self.ui.groupBox_2.setFocusPolicy(Qt.NoFocus) self.ui.btnAdd.setFocusPolicy(Qt.NoFocus) self.ui.btnAdd_2.setFocusPolicy(Qt.NoFocus) self.ui.btnAdd_3.setFocusPolicy(Qt.NoFocus) self.ui.btnAdd_4.setFocusPolicy(Qt.NoFocus) self.ui.checkBox_4.setFocusPolicy(Qt.NoFocus) self.ui.checkBox_4.setChecked(False) self.ui.btnClose.setFocusPolicy(Qt.NoFocus) self.ui.btnClose.clicked.connect(self.hide) self.ui.btnClose.clicked.connect(self.slot_click_close) self.ui.btnAdd.clicked.connect(self.slot_click_login) self.ui.btnAdd_2.clicked.connect(self.slot_click_adduser) self.ui.lesource_2.setEchoMode(QLineEdit.Password) self.ui.lesource_4.setEchoMode(QLineEdit.Password) self.ui.lesource_2.setContextMenuPolicy(Qt.NoContextMenu) self.ui.lesource_4.setContextMenuPolicy(Qt.NoContextMenu) self.ui.lesource.textChanged.connect(self.slot_le_input) self.ui.lesource_2.textChanged.connect(self.slot_le_input2) self.ui.lesource_3.textChanged.connect(self.slot_le_input3) self.ui.lesource_4.textChanged.connect(self.slot_le_input4) self.ui.lesource_5.textChanged.connect(self.slot_le_input5) self.ui.text10.clicked.connect(self.find_password_suc) self.ui.lesource.setMaxLength(30) self.ui.lesource_2.setMaxLength(30) self.ui.lesource_3.setMaxLength(30) self.ui.lesource_4.setMaxLength(30) self.ui.lesource_5.setMaxLength(30) self.timer = QTimer(self) # 初始化一个定时器 #self.ui.lesource_8.setMaxLength(16) #self.ui.lesource_9.setMaxLength(16) self.ui.tips_user_password.setText("用户名或密码错误") self.ui.tips_user_password.setAlignment(Qt.AlignCenter) #设置字体居中 self.ui.btnAdd.setText("立即登录") self.ui.btnAdd_2.setText("注册新账号") self.ui.btnAdd_3.setText("登录") self.ui.btnAdd_3.clicked.connect(self.slot_login) self.ui.btnAdd_4.setText("立即注册") self.ui.btnAdd_4.clicked.connect(self.slot_adduser) self.ui.lesource.setPlaceholderText("请输入您的用户名") self.ui.usr_icon.setStyleSheet( "QWidget{background-image:url('res/username.png');background-color:#ffffff;border:0px}" ) self.ui.lesource_2.setPlaceholderText("请输入密码") self.ui.password_icon.setStyleSheet( "QWidget{background-image:url('res/password.png');background-color:#ffffff;border:0px}" ) self.ui.lesource_3.setPlaceholderText("请输入用户名") self.ui.creat_usr_icon.setStyleSheet( "QWidget{background-image:url('res/username.png');background-color:#ffffff;border:0px}" ) self.ui.lesource_4.setPlaceholderText("请输入密码") self.ui.create_password_icon.setStyleSheet( "QWidget{background-image:url('res/password.png');background-color:#ffffff;border:0px}" ) self.ui.lesource_5.setPlaceholderText("请输入注册邮箱") self.ui.create_exmail_icon.setStyleSheet( "QWidget{background-image:url('res/exmail.png');background-color:#ffffff;border:0px}" ) #self.ui.lesource_8.setPlaceholderText("记住密码") #self.ui.lesource_9.setPlaceholderText("自动登录") #self.ui.text1.setText("登录软件中心:") #self.ui.text1.setStyleSheet("color:#ff6600;") #self.ui.text1.setStyleSheet("color:1997FAB;") # self.ui.text2.setText("用户名:") # self.ui.text3.setText("密 码:") # self.ui.text4.setText("用户名:") # self.ui.text5.setText("密 码:")zh # self.ui.text6.setText("邮 箱:") self.ui.text7.setText("是否是开发者") self.ui.text8.setText("记住密码") self.ui.text9.setText("自动登录") self.ui.text10.setText("找回密码") self.ui.soft_linedit.setText("软件商店") self.ui.spot_linedit.setText("·") self.ui.login_linedit.setText("登录") self.ui.register_newuser.setText("注册新账户") self.ui.register_newuser.hide() self.ui.groupBox_2.hide() self.ui.log_png.setStyleSheet( "QWidget{background-image:url('res/smalllogo.png');}") self.ui.soft_linedit.setStyleSheet( "QLabel{font-weight:bold;color:#666666}") self.ui.login_linedit.setStyleSheet("QLabel{color:#666666}") self.ui.spot_linedit.setStyleSheet("QLabel{color:#666666}") self.ui.register_newuser.setStyleSheet("QLabel{color:#666666}") self.ui.spot_linedit.setStyleSheet("QLabel{font-weight:bold;}") #self.ui.sourceWidget.setStyleSheet("QWidget{border:0px solid #c0d3dd;border-radius:2px;color:#0763ba;background:#ebf2f9;}") #self.ui.sourceWidget.setStyleSheet("QPushButton{border:1px solid #026c9e;color:#ebf2f9;}") self.ui.sourceWidget.setStyleSheet( ".QWidget{border:1px solid #cccccc;background-color:#ffffff}") self.ui.topWidget.setStyleSheet( "QWidget{border:0px;background-color:#eff2f6}") self.ui.clickWidget.setStyleSheet( "QWidget{border:0px solid #c0d3dd;border-radius:2px;color:#0763ba;background:#c0d3dd;}" ) self.ui.tips_user_password.setStyleSheet( "QLabel{background-color:#fffae1;font-size:12px;border:1px solid #fff0d4;color:#ff5b50;}" ) #self.ui.sourceWidget.setStyleSheet("color:#ebf2f9i;") #self.ui.btnAds.setStyleSheet("QPushButton{color:white;border:-2px;background-image:url('res/wincard-run-btn-1.png');}") #self.ui.btnAds.setStyleSheet("QPushButton{color:white;border:-2px;background-image:url('res/wincard-un-btn-2.png');}") #self.ui.text1.setText("登录软件中心:") #self.ui.text1.setStyleSheet("color:#ff6600;") self.ui.text1.setStyleSheet("color:1997FAB;") self.ui.bg.setStyleSheet( "QLabel{border:0px solid #c0d3dd;border-radius:2px;color:#026c9e;background:#ebf2f9;}" ) #self.ui.bg.setStyleSheet("QLabel{border:0px solid #026c9e;border-radius:1px;color:#ebf2f9;font-size:13px;background-image:url('res/1.png');}") self.ui.btnClose.setStyleSheet( "QPushButton{background-image:url('res/close-1.png');border:0px;}QPushButton:hover{background:url('res/close-2.png');background-color:#bb3c3c;}QPushButton:pressed{background:url('res/close-3.png');background-color:#bb3c3c;}" ) #self.ui.btnClose.setStyleSheet("QPushButton{background-image:url('res/delete-normal.png');border:0px;}QPushButton:hover{background:url('res/delete-pressed.png');}QPushButton:pressed{background:url('res/delete-pressed.png');}") #self.ui.lesource.setStyleSheet("QLineEdit{border:0px solid #6BB8DD;border-radius:1px;color:#497FAB;font-size:13px;}") self.ui.groupBox.setStyleSheet("QGroupBox{border:0px;}") self.ui.groupBox_2.setStyleSheet("QGroupBox{border:0px;}") self.ui.btnAdd.setStyleSheet( "QPushButton{border:0px;font-size:12px;no-repeat center left;color:#2d8ae1}QPushButton:hover{font-size:13px;color:#2d8ae1;}" ) self.ui.btnAdd_2.setStyleSheet( "QPushButton{border:0px;font-size:12px;no-repeat center left;color:#2d8ae1}QPushButton:hover{font-size:13px;color:#2d8ae1;}" ) self.ui.lesource_parent.setStyleSheet( "QWidget{border:1px solid #bec2cc;border-radius:2px;color:#997FAB;font-size:12px;}QWidget:hover{border:1px solid #2d8ae1;}" ) self.ui.lesource_2_parent.setStyleSheet( "QWidget{border:1px solid #bec2cc;border-radius:2px;color:#997FAB;font-size:12px;}QWidget:hover{border:1px solid #2d8ae1;}" ) self.ui.lesource_3_parent.setStyleSheet( "QWidget{border:1px solid #bec2cc;border-radius:2px;color:#997FAB;font-size:12px;}QWidget:hover{border:1px solid #2d8ae1;}" ) self.ui.lesource_4_parent.setStyleSheet( "QWidget{border:1px solid #bec2cc;border-radius:2px;color:#997FAB;font-size:12px;}QWidget:hover{border:1px solid #2d8ae1;}" ) self.ui.lesource_5_parent.setStyleSheet( "QWidget{border:1px solid #bec2cc;border-radius:2px;color:#997FAB;font-size:12px;}QWidget:hover{border:1px solid #2d8ae1;}" ) self.ui.lesource.setStyleSheet( "QLineEdit{border:0px;border-radius:2px;color:#aaaaaa;font-size:12px;}QLineEdit:pressed{color:#000000;}" ) self.ui.lesource_2.setStyleSheet( "QLineEdit{border:0px;border-radius:2px;color:#aaaaaa;font-size:12px;}QLineEdit:pressed{color:#000000;}" ) self.ui.lesource_3.setStyleSheet( "QLineEdit{border:0px;border-radius:2px;color:#aaaaaa;font-size:12px;}QLineEdit:pressed{color:#000000;}" ) self.ui.lesource_4.setStyleSheet( "QLineEdit{border:0px;border-radius:2px;color:#aaaaaa;font-size:12px;}QLineEdit:pressed{color:#000000;}" ) self.ui.lesource_5.setStyleSheet( "QLineEdit{border:0px;border-radius:2px;color:#aaaaaa;font-size:12px;}QLineEdit:pressed{color:#000000;}" ) #self.ui.lesource_8.setStyleSheet("QLineEdit{border:1px solid #6BB8DD;border-radius:2px;color:#997FAB;font-size:13px;}") #self.ui.lesource_9.setStyleSheet("QLineEdit{border:1px solid #6BB8DD;border-radius:2px;color:#997FAB;font-size:13px;}") # self.ui.btnAdd_3.setStyleSheet("QPushButton{color:white;border:0px;border-radius:4px;backgroound-color:#2d8ae1;}QPushButton:hover{border:0px;}QPushButton:pressed{border:0px;}") self.ui.btnAdd_3.setStyleSheet( "QPushButton{background-color:#2d8ae1;border:0px;font-size:16px;border-radius:4px;color:#ffffff}QPushButton:hover{background-color:#3580c4;border:0px;border-radius:4px;font-size:16px;color:#ffffff}" ) # self.ui.btnAdd_4.setStyleSheet("QPushButton{color:white;border:0px;border-radius:4px;background-image:url('res/click-up-btn-2.png');}QPushButton:hover{border:0px;background-image:url('res/click-up-btn-3.png');}QPushButton:pressed{border:0px;background-image:url('res/click-up-btn-1.png');}") self.ui.btnAdd_4.setStyleSheet( "QPushButton{background-color:#2d8ae1;border:0px;font-size:16px;border-radius:4px;color:#ffffff}QPushButton:hover{background-color:#3580c4;border:0px;border-radius:4px;font-size:16px;color:#ffffff}" ) self.ui.text10.setStyleSheet( "QPushButton{border:0px;font-size:12px;color:#2d8ae1;}QPushButton:hover{border:0px;font-size:13px;color:#2d8ae1;}QPushButton:pressed{border:0px;font-size:13px;color:#2d8ae1;}" ) self.ui.text10.setFocusPolicy(Qt.NoFocus) if (Globals.SET_REM): self.ui.lesource.setText(Globals.OS_USER) self.ui.lesource_2.setText(Globals.PASSWORD) self.ui.checkBox_5.setChecked(True)
class Database: def __init__(self): self.updatecount = 0 srcFile = UBUNTUKYLIN_DATA_UKSCDB_PATH #源数据库 destFile = UBUNTUKYLIN_CACHE_UKSCDB_PATH #目标数据库 # no cache file, copy if not os.path.exists(destFile): if not os.path.exists(srcFile): if (Globals.DEBUG_SWITCH): LOG.error("error with db file") return open(destFile, "wb").write(open(srcFile, "rb").read()) self.connect = sqlite3.connect(destFile, timeout=30.0, check_same_thread=False) self.cursor = self.connect.cursor() self.cat_list = [] # cache file need update, copy if self.is_cachedb_need_update(): open(destFile, "wb").write(open(srcFile, "rb").read()) # piston remoter to ukscs self.premoter = PistonRemoter(service_root=UBUNTUKYLIN_SERVER) #___________________________add by zhangxin for xapiandb update___________________________# xapian_srcFile = XAPIAN_DB_SOURCE_PATH xapian_destFile = os.path.join(UKSC_CACHE_DIR, "xapiandb") # no cache file, copy if not os.path.exists(xapian_destFile): if not os.path.exists(xapian_srcFile): if (Globals.DEBUG_SWITCH): LOG.error( "No xapiandb source in /usr/share/ubuntu-kylin-software-center/data/,please reinstall it" ) return copytree(xapian_srcFile, xapian_destFile) if (Globals.DEBUG_SWITCH): LOG.debug("Xapiandb has been copy to cache") # cache xapiandb need update, copy if self.is_xapiancachedb_need_update(): rmtree(xapian_destFile) copytree(xapian_srcFile, xapian_destFile) if (Globals.DEBUG_SWITCH): LOG.debug("cache xapiandb versin updated") def query_categories(self): try: lock.acquire(True) self.cursor.execute("select * from category") res = self.cursor.fetchall() finally: lock.release() # print "query_categories:",len(res),res return res def query_category_apps(self, cate_name): al = '' sql = "select id from category where name='%s'" try: lock.acquire(True) self.cursor.execute(sql % cate_name) res = self.cursor.fetchall() finally: lock.release() cateid = '' for i in res: cateid = i[0] sql = "select id,categories from application" try: lock.acquire(True) self.cursor.execute(sql) res = self.cursor.fetchall() finally: lock.release() for i in res: aid = i[0] cstring = i[1] cs = cstring.split(',') for c in cs: #将unicode中的非数字的部分去掉,数据库被不正确修改后的错误 if c.isdigit(): #c = list(filter(str.isdigit, c.encode("utf-8"))) c = re.sub("\D", "", c) c = c.encode("utf-8") if (int(cateid) == int(c)): al += str(aid) al += ',' al = al[:-1] sql = "select app_name,display_name_cn from application where id in (%s) order by rating_avg DESC" try: lock.acquire(True) self.cursor.execute(sql % al) res = self.cursor.fetchall() finally: lock.release() # for a in res: # print a[0]," ",a[1] return res #return as (display_name, summary, description, rating_average,rating_total,review_total,download_total) def query_application(self, pkgname): try: lock.acquire(True) self.cursor.execute(QUERY_APP % (pkgname)) res = self.cursor.fetchall() finally: lock.release() # print "query_application:",pkgname,len(res),res if len(res) == 0: return [] else: return res[0] #return as (display_name, app_name) def query_applications(self): try: lock.acquire(True) self.cursor.execute(QUERY_APPS) res = self.cursor.fetchall() finally: lock.release() # print "query_application:",pkgname,len(res),res if len(res) == 0: return [] else: return res def update_app_rnr(self, pkgname, rating_average, rating_total, review_total, download_total=0): if (Globals.DEBUG_SWITCH): print(("update_app_rnr:", self.updatecount, pkgname, rating_average, rating_total, review_total, download_total)) try: lock.acquire(True) self.cursor.execute(UPDATE_APP_RNR % (rating_average, rating_total, review_total, download_total, pkgname)) self.connect.commit() finally: lock.release() #res = self.cursor.fetchall() #print "update_app_rnr:",len(res),res self.updatecount += 1 return True #---------------------------------0.3---------------------------------- expiredict = {} # check the ~/.cache/uksc/uksc.db version, and copy /usr/share/u../data/uksc.db to replace it def is_cachedb_need_update(self): srcFile = UBUNTUKYLIN_DATA_UKSCDB_PATH connectsrc = sqlite3.connect(srcFile, timeout=30.0, check_same_thread=False) cursorsrc = connectsrc.cursor() try: lock.acquire(True) self.cursor.execute( "select count(*) from sqlite_master where type='table' and name='dict'" ) res = self.cursor.fetchall() finally: lock.release() dictcount = '' for item in res: dictcount = item[0] if (dictcount == 0): return True try: lock.acquire(True) self.cursor.execute("select value from dict where key='dbversion'") res = self.cursor.fetchall() finally: lock.release() olddbversion = '' for item in res: olddbversion = int(item[0]) try: lock.acquire(True) cursorsrc.execute("select value from dict where key='dbversion'") res = cursorsrc.fetchall() finally: lock.release() newdbversion = '' for item in res: newdbversion = int(item[0]) if (newdbversion > olddbversion): return True return False #------------------------------add by zhangxin--------------------------------------------- def is_xapiancachedb_need_update(self): xapian_srcFile = XAPIAN_DB_SOURCE_PATH xapian_destFile = os.path.join(UKSC_CACHE_DIR, "xapiandb") try: src_xapiandb = xapian.Database(xapian_srcFile) new_enquire = xapian.Enquire(src_xapiandb) new_query = xapian.Query("the_#ukxapiandb#_version") new_enquire.set_query(new_query) new_matches = new_enquire.get_mset(0, 1) for new_item in new_matches: new_doc = new_item.document if new_doc.get_data() == "XAPIANDB_VERSION": new_version = new_doc.get_value( 1) #valueslot:1 xapiandb version des_xapiandb = xapian.Database(xapian_destFile) old_enquire = xapian.Enquire(des_xapiandb) old_query = xapian.Query("the_#ukxapiandb#_version") old_enquire.set_query(old_query) old_matches = old_enquire.get_mset(0, 1) for old_item in old_matches: old_doc = old_item.document old_version = old_doc.get_value( 1) #valueslot:1 xapiandb version #if (Globals.DEBUG_SWITCH): print(("old xapiandb version:", old_version, " new xapiandb version:", new_version)) except: return True else: if (new_version > old_version): return True else: return False def get_pagecount_by_pkgname(self, package_name): try: lock.acquire(True) self.cursor.execute( "select review_total from application where app_name=?", (package_name, )) res = self.cursor.fetchall() finally: lock.release() for item in res: review_total = item[0] if (review_total == None): review_total = 0 return review_total / 10 + 1 def get_review_by_pkgname(self, package_name, page): # get application id try: lock.acquire(True) self.cursor.execute("select id from application where app_name=?", (package_name, )) res = self.cursor.fetchall() finally: lock.release() aid = '' for item in res: aid = str(item[0]) if not aid: return [] # get review count try: lock.acquire(True) self.cursor.execute("select count(*) from review where aid_id=?", (aid, )) res = self.cursor.fetchall() finally: lock.release() count = '' for item in res: count = item[0] if (page == 1): # empty cache, download page 1 if (count == 0): reviews = self.premoter.get_reviews(package_name, 0, 10) if False != reviews: review_total = -1 for review in reviews: id = str(review.id) review_total = review.aid['review_total'] user_display = review.user_display content = review.content date = str(review.date) date = date.replace('T', ' ') date = date.replace('Z', '') try: lock.acquire(True) self.cursor.execute( "insert into review values(?,?,'',?,?,?,'zh_CN','',0,0)", (id, aid, content, user_display, date)) self.connect.commit() finally: lock.release() if (review_total != ''): try: lock.acquire(True) self.cursor.execute( "update application set review_total=? where id=?", (review_total, aid)) self.connect.commit() finally: lock.release() # normal init, check and download newest reviews else: # get newest review's id from local cache try: lock.acquire(True) self.cursor.execute( "select id from review where aid_id=? order by date DESC limit 0,1", (aid, )) res = self.cursor.fetchall() finally: lock.release() id = '' for item in res: id = item[0] # find newest reviews from server startpage = 0 loop = True while loop: reviews = self.premoter.get_reviews( package_name, startpage, 10) if False != reviews: review_total = -1 # 0 review on server. break if (len(reviews) == 0): break for review in reviews: rid = review.id review_total = review.aid['review_total'] user_display = review.user_display content = review.content date = str(review.date) date = date.replace('T', ' ') date = date.replace('Z', '') # download if (rid != id): try: lock.acquire(True) self.cursor.execute( "insert or ignore into review values(?,?,'',?,?,?,'zh_CN','',0,0)", (rid, aid, content, user_display, date)) self.connect.commit() finally: lock.release() # end download else: # stop 'while' loop = False # break 'for' break # cannot find the local newest review from server, break if (startpage > (review_total / 10 + 1)): # stop 'while' loop = False # break 'for' break if (review_total != ''): try: lock.acquire(True) self.cursor.execute( "update application set review_total=? where id=?", (review_total, aid)) self.connect.commit() finally: lock.release() startpage += 1 self.connect.commit() else: loop = False else: # review not enough, download if (count < page * 10): start = (page - 1) * 10 reviews = self.premoter.get_reviews(package_name, start, 10) if False != reviews: review_total = '' for review in reviews: id = str(review.id) review_total = review.aid['review_total'] user_display = review.user_display content = review.content date = str(review.date) date = date.replace('T', ' ') date = date.replace('Z', '') # ignore the same review by id try: lock.acquire(True) self.cursor.execute( "insert or ignore into review values(?,?,'',?,?,?,'zh_CN','',0,0)", (id, aid, content, user_display, date)) self.connect.commit() finally: lock.release() if (review_total != ''): try: lock.acquire(True) self.cursor.execute( "update application set review_total=? where id=?", (review_total, aid)) self.connect.commit() finally: lock.release() # all download check over, return reviews to show limit = (page - 1) * 10 try: lock.acquire(True) self.cursor.execute( "select id, aid_id, content, user_display, date, language, up_total, down_total from review where aid_id=? order by date DESC limit ?,10", (aid, limit)) res = self.cursor.fetchall() finally: lock.release() reviews = [] for item in res: review = Review(package_name) review.id = item[0] review.aid = item[1] review.content = item[2] review.user_display = item[3] review.date = item[4] review.language = item[5] review.up_total = item[6] review.down_total = item[7] reviews.append(review) return reviews def get_pointout_is_show(self): try: lock.acquire(True) self.cursor.execute("select value from dict where key=?", ('pointout', )) res = self.cursor.fetchall() finally: lock.release() for item in res: isshow = item[0] return isshow def set_pointout_is_show(self, flag): value = '' if (flag == True): value = 'True' else: value = 'False' try: lock.acquire(True) self.cursor.execute("update dict set value=? where key='pointout'", (value, )) self.connect.commit() finally: lock.release() def get_pointout_apps(self): try: lock.acquire(True) self.cursor.execute( "select app_name,rank_pointout from rank,application where rank_pointout!=0 and rank.aid_id=application.id order by rank_pointout" ) res = self.cursor.fetchall() finally: lock.release() pointouts = [] for item in res: app_name = item[0] rank_pointout = item[1] pointouts.append((app_name, rank_pointout)) return pointouts def get_recommend_apps(self): #self.cursor.execute("select app_name,rank_recommend from rank,application where rank_recommend!=0 and rank.aid_id=application.id order by rank_recommend") #self.cursor.execute("select app_name,rank from application where application.rank!=0 order by rank") #res = self.cursor.fetchall() recommends = [] #for item in res: # app_name = item[0] # print "ddddddddddddddddd",app_name,item[1] # rank_recommend = item[1] # recommends.append((app_name, rank_recommend)) recommends.append(("youker-assistant", "1")) recommends.append(("com.tencent.mobileqq", "1")) recommends.append(("com.qqgame.hlddz", "1")) recommends.append(("cn.kuwo.player", "1")) recommends.append(("atom", "3")) recommends.append(("google-chrome-stable", "3")) recommends.append(("sogoupinyin", "1")) recommends.append(("skypeforlinux", "6")) recommends.append(("lovewallpaper", "3")) recommends.append(("franz", "3")) recommends.append(("youdao-dict", "3")) recommends.append(("foxitreader", "8")) recommends.append(("ppsspp", "3")) recommends.append(("virtualbox", "6")) recommends.append(("thunderbird", "7")) recommends.append(("openshot", "12")) recommends.append(("firefox", "11")) recommends.append(("wireshark", "9")) recommends.append(("librecad", "12")) recommends.append(("flashplugin-installer", "13")) # recommends.append(("brasero","13")) recommends.append(("uget", "5")) recommends.append(("calibre", "10")) recommends.append(("gimp", "3")) recommends.append(("gtk-recordmydesktop", "3")) recommends.append(("transgui", "3")) recommends.append(("pluma", "3")) recommends.append(("gnome-calculator", "3")) #recommends.append(("virtualbox","4")) # youker-assistant gimp sogoupinyin virtualbox wine playonlinux freecad return recommends def get_game_apps(self): #self.cursor.execute("select app_name,rank_recommend from rank,application where rank_recommend!=0 and rank.aid_id=application.id order by rank_recommend") #self.cursor.execute("select app_name,rank from application where application.rank!=0 order by rank") #res = self.cursor.fetchall() recommends = [] #for item in res: # app_name = item[0] # print "ddddddddddddddddd",app_name,item[1] # rank_recommend = item[1] # recommends.append((app_name, rank_recommend)) recommends.append(("playonlinux", "2")) recommends.append(("steam-launcher", "3")) recommends.append(("ppsspp", "3")) recommends.append(("crossover", "3")) recommends.append(("wine1.6", "1")) recommends.append(("supertuxkart", "4")) recommends.append(("kodi", "1")) recommends.append(("aisleriot", "1")) recommends.append(("gnome-chess", "7")) recommends.append(("supertux", "7")) recommends.append(("gnchess", "11")) recommends.append(("flightgear", "10")) recommends.append(("kmahjongg", "6")) recommends.append(("frozen-bubble", "9")) recommends.append(("gnome-hearts", "8")) recommends.append(("gnome-mines", "13")) recommends.append(("smplayer", "14")) recommends.append(("openrocket", "5")) recommends.append(("gnome-mahjongg", "12")) recommends.append(("funnyboat", "12")) recommends.append(("kylin-video", "12")) # youker-assistant gimp sogoupinyin virtualbox wine playonlinux freecad return recommends def get_necessary_apps(self): recommends = [] recommends.append(("opera", "2")) recommends.append(("filezilla", "2")) recommends.append(("remmina", "3")) recommends.append(("obs-studio", "3")) recommends.append(("freecad", "4")) recommends.append(("unetbootin", "5")) recommends.append(("codeblocks", "6")) recommends.append(("chromium-browser", "8")) recommends.append(("transmission", "7")) recommends.append(("deluge", "8")) recommends.append(("hardinfo", "9")) recommends.append(("nautilus", "10")) recommends.append(("bluefish", "11")) recommends.append(("gnome-screenshot", "12")) recommends.append(("blender", "13")) recommends.append(("xmind", "14")) recommends.append(("vim", "14")) recommends.append(("midori", "14")) recommends.append(("notepadqq", "8")) recommends.append(("gnome-disk-utility", "14")) recommends.append(("shotcut", "14")) recommends.append(("gparted", "14")) return recommends # def get_ratingrank_apps(self): # #self.cursor.execute("select app_name,rank_rating from rank,application where rank_rating!=0 and rank.aid_id=application.id order by rank_rating") # #res = self.cursor.fetchall() # #ratingranks = [] # #for item in res: # # app_name = item[0] # # rank_rating = item[1] # # ratingranks.append((app_name, rank_rating)) # #return ratingranks # recommends = [] # #recommends.append(("youker-assistant","1")) # recommends.append(("vlc","2")) # recommends.append(("synaptic","3")) # recommends.append(("gparted","3")) # recommends.append(("fcitx","3")) # recommends.append(("qtcreator","3")) # recommends.append(("rar","3")) # recommends.append(("shotwell","3")) # recommends.append(("stellarium","3")) # recommends.append(("ubuntu-restricted-extras","3")) # recommends.append(("stardict","3")) # recommends.append(("vim","14")) # recommends.append(("kylin-video","12")) # recommends.append(("gnome-screenshot","12")) # recommends.append(("empire","12")) # return recommends def update_app_ratingavg(self, app_name, ratingavg, ratingtotal): try: lock.acquire(True) self.cursor.execute( "update application set rating_avg=?,rating_total=? where app_name=?", (ratingavg, ratingtotal, app_name)) self.connect.commit() finally: lock.release() def update_app_downloadtotal(self, app_name, download_total=''): try: lock.acquire(True) if (download_total): self.cursor.execute( "update application set download_total=? where app_name=?", (download_total, app_name)) else: self.cursor.execute( "update application set download_total=download_total+1 where app_name=?", (app_name, )) self.connect.commit() finally: lock.release() def get_app_downloadtotal(self, app_name): try: lock.acquire(True) self.cursor.execute( "select download_total from application where app_name=?", (app_name, )) res = self.cursor.fetchall() finally: lock.release() return res #------------add by kobe for windows replace------------ def search_name_and_categories_record(self): try: lock.acquire(True) self.cursor.execute(QUERY_NAME_CATEGORIES) res = self.cursor.fetchall() finally: lock.release() if len(res) == 0: return [] else: return res #------------add by kobe for windows replace------------ def search_app_display_info(self, categories): try: lock.acquire(True) self.cursor.execute(QUERY_APP_ACCORD_CATEGORIES % (categories)) res = self.cursor.fetchall() finally: lock.release() if len(res) == 0: return [] else: return res #------------add by kobe for windows replace------------ def update_exists_data(self, exists, id): try: lock.acquire(True) self.cursor.execute(UPDATE_EXISTS % (exists, id)) self.connect.commit() finally: lock.release() def need_do_sourcelist_update(self): try: lock.acquire(True) self.cursor.execute("select value from dict where key=?", ('sourcelist_need_update', )) res = self.cursor.fetchall() finally: lock.release() for item in res: re = item[0] return re def set_update_sourcelist_false(self): try: lock.acquire(True) self.cursor.execute( "update dict set value=? where key='sourcelist_need_update'", ("False", )) self.connect.commit() finally: lock.release() #-------------kydroid APK ---------------- def query_apk_applications(self): try: lock.acquire(True) self.cursor.execute( "select app_name,display_name_cn,summary,description,rating_avg,rating_total,review_total from application where id > 3410 and categories = '17'" ) res = self.cursor.fetchall() finally: lock.release() # print "query_application:",pkgname,len(res),res if len(res) == 0: return [] else: return res
class SilentProcess(multiprocessing.Process): def __init__(self, squeue): super(SilentProcess, self).__init__() multiprocessing.Process.__init__(self) self.daemon = True self.squeue = squeue self.destFile = os.path.join(UKSC_CACHE_DIR, "uksc.db") self.connect = sqlite3.connect(self.destFile, timeout=30.0, check_same_thread=False) self.cursor = self.connect.cursor() self.premoter = PistonRemoter(service_root=UBUNTUKYLIN_SERVER) # self.httpmodel = HttpDownLoad() def run(self): while True: if self.squeue.empty(): time.sleep(1) continue item = self.squeue.get_nowait() if (Globals.DEBUG_SWITCH): print("silent process get one workitem : ", item.funcname) try: #if no network the process will be crashed if item.funcname == "get_all_ratings": self.get_all_ratings() elif item.funcname == "submit_pingback_main": self.submit_pingback_main() elif item.funcname == "submit_pingback_app": self.submit_pingback_app(item.kwargs) elif item.funcname == "get_all_categories": self.get_all_categories() elif item.funcname == "get_all_rank_and_recommend": self.get_all_rank_and_recommend() elif item.funcname == "get_newer_application_info": self.get_newer_application_info() elif item.funcname == "get_newer_application_icon": self.get_newer_application_icon() elif item.funcname == "update_xapiandb": self.update_xapiandb(item.kwargs) elif item.funcname == "get_newer_application_ads": self.get_newer_application_ads() elif item.funcname == "get_newer_application_screenshots": self.get_newer_application_screenshots() except Exception as e: if (Globals.DEBUG_SWITCH): print("silent process exception:", e) # elif item.funcname == "download_images": # self.download_images() # update rating_avg and rating_total in cache db from server def get_all_ratings(self): reslist = self.premoter.get_all_ratings() if False != reslist: for rating in reslist: app_name = rating['app_name'] rating_avg = str(rating['rating_avg']) rating_total = str(rating['rating_total']) try: lock.acquire(True) sql = "update application set rating_total=?,rating_avg=? where app_name=?" self.cursor.execute(sql, (rating_total, rating_avg, app_name)) finally: lock.release() try: lock.acquire(True) self.connect.commit() finally: lock.release() #self.connect.commit() if (Globals.DEBUG_SWITCH): print("all ratings and rating_total update over : ", len(reslist)) else: if (Globals.DEBUG_SWITCH): print("Failed to get all ratings and rating_total") # submit pingback-main to server def submit_pingback_main(self): machine = get_machine_id() distro = get_distro_info()[0] version_os = get_distro_info()[1] version_uksc = get_uksc_version() res = self.premoter.submit_pingback_main(machine, distro, version_os, version_uksc) return res # submit pingback-app to server def submit_pingback_app(self, kwargs): app_name = kwargs["app_name"] isrcm = kwargs["isrcm"] user = kwargs["user"] machine = get_machine_id() res = self.premoter.submit_pingback_app(app_name, machine, isrcm, user) return res # get all categories data from server def get_all_categories(self): reslist = self.premoter.get_all_categories() if False != reslist: for category in reslist: cid = category['id'] name = category['name'] display_name = category['display_name'] priority = category['priority'] sql = "select count(*) from category where id=?" try: lock.acquire(True) self.cursor.execute(sql, (cid, )) res = self.cursor.fetchall() finally: lock.release() isexist = '' for item in res: isexist = item[0] if (isexist == 1): # id exist, update sql = "update category set name=?,display_name=?,priority=? where id=?" try: lock.acquire(True) self.cursor.execute( sql, (name, display_name, priority, cid)) self.connect.commit() finally: lock.release() else: # id not exist, insert sql = "insert into category(id,name,display_name,priority,visible) values(?,?,?,?,1)" try: lock.acquire(True) self.cursor.execute( sql, (cid, name, display_name, priority)) self.connect.commit() finally: lock.release() self.connect.commit() if (Globals.DEBUG_SWITCH): print("all categories update over : ", len(reslist)) else: if (Globals.DEBUG_SWITCH): print("Failed to all categories") # get all rank and recommend data from server def get_all_rank_and_recommend(self): reslist = self.premoter.get_all_rank_and_recommend() if False != reslist: sql = "delete from rank" try: lock.acquire(True) self.cursor.execute(sql) self.connect.commit() finally: lock.release() for rank in reslist: rid = rank['id'] aid = rank['aid']['id'] rank_rating = rank['rank_rating'] rank_download = rank['rank_download'] rank_recommend = rank['rank_recommend'] rank_pointout = rank['rank_pointout'] sql = "insert into rank(id,aid_id,rank_rating,rank_download,rank_recommend,rank_pointout) values(?,?,?,?,?,?)" try: lock.acquire(True) self.cursor.execute(sql, (rid, aid, rank_rating, rank_download, rank_recommend, rank_pointout)) self.connect.commit() finally: lock.release() if (Globals.DEBUG_SWITCH): print("all rank and recommend update over : ", len(reslist)) else: if (Globals.DEBUG_SWITCH): print("Failed to all rank and recommend") # get newer application info from server def get_newer_application_info(self): # get application info last update date last_update_date = '' try: lock.acquire(True) self.cursor.execute( "select value from dict where key='appinfo_updatetime'") res = self.cursor.fetchall() finally: lock.release() for item in res: last_update_date = item[0] reslist = self.premoter.get_newer_application_info(last_update_date) if False != reslist: size = len(reslist) if (size > 0): # update application info to cache db for app in reslist: aid = app['id'] app_name = app['app_name'] display_name = app['display_name'] display_name_cn = app['display_name_cn'] categories = app['categories'] summary = app['summary'] description = app['description'] command = app['command'] rating_avg = app['rating_avg'] rating_total = app['rating_total'] review_total = app['review_total'] download_total = app['download_total'] # if summary == '': # summary = None # if description == '': # description = None # if command == '': # command = None sql = "select count(*) from application where id=?" try: lock.acquire(True) self.cursor.execute(sql, (aid, )) res = self.cursor.fetchall() finally: lock.release() isexist = '' for item in res: isexist = item[0] if (isexist == 1): # id exist, update sql = "update application set app_name=?,display_name=?,display_name_cn=?,categories=?,summary=?,description=?,command=?,rating_avg=?,rating_total=?,review_total=?,download_total=? where id=?" try: lock.acquire(True) self.cursor.execute( sql, (app_name, display_name, display_name_cn, categories, summary, description, command, rating_avg, rating_total, review_total, download_total, aid)) self.connect.commit() finally: lock.release() else: # id not exist, insert sql = "insert into application(id,app_name,display_name,display_name_cn,categories,summary,description,command,rating_avg,rating_total,review_total,download_total) values(?,?,?,?,?,?,?,?,?,?,?,?)" try: lock.acquire(True) self.cursor.execute( sql, (aid, app_name, display_name, display_name_cn, categories, summary, description, command, rating_avg, rating_total, review_total, download_total)) self.connect.commit() finally: lock.release() # set application info last update date updatetime = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.localtime()) try: lock.acquire(True) self.cursor.execute("update dict set value=? where key=?", (updatetime, 'appinfo_updatetime')) self.connect.commit() finally: lock.release() #self.connect.commit() if (Globals.DEBUG_SWITCH): print("all newer application info update over : ", len(reslist)) else: if (Globals.DEBUG_SWITCH): print("failed to get newer application info") # def download_images(self): # requestData = "http://service.ubuntukylin.com:8001/uksc/download/?name=uk-win.zip" # url = QUrl(requestData) # self.httpmodel.sendDownLoadRequest(url) def get_newer_application_icon(self): # get application icon last update date last_update_date = '' try: lock.acquire(True) self.cursor.execute( "select value from dict where key='appicon_updatetime'") res = self.cursor.fetchall() finally: lock.release() for item in res: last_update_date = item[0] reslist = self.premoter.get_newer_application_icon(last_update_date) if False != reslist: size = len(reslist) if (size > 0): # update application icon to cache icons/ for app in reslist: app_name = app['name'] app_path = app['path'] iconfile = UBUNTUKYLIN_CACHE_ICON_PATH + app_name try: icon_rul = UK_APP_ICON_URL % { 'pkgname': app_path, } urlFile = urllib.request.urlopen(icon_rul) rawContent = urlFile.read() if rawContent: localFile = open(iconfile, "wb") localFile.write(rawContent) localFile.close() except urllib.error.HTTPError as e: if (Globals.DEBUG_SWITCH): print(e.code) except urllib.error.URLError as e: if (Globals.DEBUG_SWITCH): print(str(e)) # set application info last update date new_update_time = reslist[0]['modify_time'] try: lock.acquire(True) self.cursor.execute( "update dict set value=? where key=?", (new_update_time, 'appicon_updatetime')) self.connect.commit() finally: lock.release() if (Globals.DEBUG_SWITCH): print("all newer application icon update over : ", len(reslist)) else: if (Globals.DEBUG_SWITCH): print("Failed to get newer application icon") def get_newer_application_screenshots(self): last_update_date = '' try: lock.acquire(True) self.cursor.execute( "select value from dict where key='appscreenshots_updatetime'") res = self.cursor.fetchall() finally: lock.release() for item in res: last_update_date = item[0] reslist = self.premoter.get_newer_application_screenshots( last_update_date) if False != reslist: size = len(reslist) if (size > 0): # update application icon to cache icons/ for app in reslist: app_name = app['name'] app_path = app['path'] iconfile = UBUNTUKYLIN_CACHE_SETSCREENSHOTS_PATH + app_name try: icon_rul = UK_APP_ICON_URL % { 'pkgname': app_path, } urlFile = urllib.request.urlopen(icon_rul) rawContent = urlFile.read() if rawContent: if not os.path.exists( UBUNTUKYLIN_CACHE_SETSCREENSHOTS_PATH): os.mkdir(UBUNTUKYLIN_CACHE_SETSCREENSHOTS_PATH) localFile = open(iconfile, "wb") localFile.write(rawContent) localFile.close() except urllib.error.HTTPError as e: if (Globals.DEBUG_SWITCH): print(e.code) except urllib.error.URLError as e: if (Globals.DEBUG_SWITCH): print(str(e)) # set application info last update date new_update_time = reslist[0]['modify_time'] try: lock.acquire(True) self.cursor.execute( "update dict set value=? where key=?", (new_update_time, 'appscreenshots_updatetime')) self.connect.commit() finally: lock.release() if (Globals.DEBUG_SWITCH): print("all newer application icon update over : ", len(reslist)) else: if (Globals.DEBUG_SWITCH): print("Failed to get newer application screenshots") def get_newer_application_ads(self): last_update_date = '' try: lock.acquire(True) self.cursor.execute( "select value from dict where key='appads_updatetime'") res = self.cursor.fetchall() finally: lock.release() for item in res: last_update_date = item[0] reslist = self.premoter.get_newer_application_ads(last_update_date) if False != reslist: sql = "delete from advertisement" self.cursor.execute(sql) size = len(reslist) if (size > 0): # update application icon to cache icons/ for app in reslist: app_name = app['name'] app_path = app['path'] ads_id = app['id'] iconfile = UBUNTUKYLIN_CACHE_SETADS_PATH + app_name + '.png' try: icon_rul = UK_APP_ICON_URL % { 'pkgname': app_path, } # print("3333333333333333333333333:",icon_rul) urlFile = urllib.request.urlopen(icon_rul) # print("2222222222222222222222222:",urlFile) rawContent = urlFile.read() if rawContent: if not os.path.exists( UBUNTUKYLIN_CACHE_SETADS_PATH): os.mkdir(UBUNTUKYLIN_CACHE_SETADS_PATH) if "uksc/ads/" in app_path: localFile = open(iconfile, "wb") localFile.write(rawContent) localFile.close() else: pass except urllib.error.HTTPError as e: if (Globals.DEBUG_SWITCH): print(e.code) except urllib.error.URLError as e: if (Globals.DEBUG_SWITCH): print(str(e)) if "uksc/ads/" in app_path: self.cursor.execute( "insert into advertisement values(?,?,?)", (ads_id, app_name, app_path)) # self.cursor.execute("insert or ignore into review values(?,?,'',?,?,?,'zh_CN','',0,0)", # (id, aid, content, user_display, date)) # set application info last update date new_update_time = reslist[0]['modify_time'] try: lock.acquire(True) self.cursor.execute( "update dict set value=? where key=?", (new_update_time, 'appads_updatetime')) self.connect.commit() finally: lock.release() if (Globals.DEBUG_SWITCH): print("all newer application icon update over : ", len(reslist)) else: if (Globals.DEBUG_SWITCH): print("Failed to get newer application icon") #*************************update for xapiandb***********************************# def update_xapiandb(self, kwargs): database = xapian.WritableDatabase(XAPIAN_DB_PATH, xapian.DB_OPEN) DB = xapian.Database(XAPIAN_DB_PATH) enquire = xapian.Enquire(database) indexer = xapian.TermGenerator() if "" == kwargs["pkgname"]: modified_num = 0 add_num = 0 xapiandb_update = "No" query_xapiandb_version = xapian.Query("the_#ukxapiandb#_version") enquire.set_query(query_xapiandb_version) matches = enquire.get_mset(0, 1) for re in matches: docid_for_xapiandb_version = re.document.get_docid() doc_for_xapiandb_version = re.document doc_data = doc_for_xapiandb_version.get_data() if (isinstance(doc_data, bytes)): doc_data = doc_data.decode(encoding='utf-8') if ("XAPIANDB_VERSION" == doc_data): the_latest_update_time = doc_for_xapiandb_version.get_value( 2) #valueslot:2 xapiandb update time if (isinstance(the_latest_update_time, bytes)): the_latest_update_time = the_latest_update_time.decode( encoding='utf-8') else: the_latest_update_time = time.strftime( '%Y-%m-%dT%H:%M:%S', time.localtime()) if (Globals.DEBUG_SWITCH): print( "Failed to get the latest update time from client xapiandb,use default time.localtime()" ) reslist = self.premoter.newerapp_for_xapianupdate( the_latest_update_time) for app in reslist: app_name = str(app["app_name"]) display_name_cn = str(app["display_name_cn"]) keywords_for_search = str(app["keywords_for_search"]) query = xapian.Query(app_name) enquire.set_query(query) doccount = DB.get_doccount() matches = enquire.get_mset(0, doccount) if matches.size() != 0: for re in matches: get_name = re.document.get_data() if (isinstance(get_name, bytes)): get_name = get_name.decode(encoding='utf-8') if get_name == app_name: docid = re.docid doc = re.document doc.clear_terms() indexer.set_document(doc) doc.add_term(app_name, 10) if keywords_for_search != "None": keywords = display_name_cn + ";" + keywords_for_search + ";" + app_name else: keywords = display_name_cn + ";" + app_name indexer.index_text(keywords, 10) try: from mmseg.search import seg_txt_search, seg_txt_2_dict for word, value in seg_txt_2_dict( keywords).items(): if word != "none": doc.add_term(word, 10) else: pass except: if (Globals.DEBUG_SWITCH): print("----No mmseg model---") database.replace_document(docid, doc) xapiandb_update = "Yes" modified_num = modified_num + 1 else: continue else: doc = xapian.Document() doc.set_data(app_name) doc.add_term(app_name, 10) indexer.set_document(doc) if keywords_for_search != "None": keywords = display_name_cn + ";" + keywords_for_search + ";" + app_name else: keywords = display_name_cn + ";" + app_name indexer.index_text(keywords, 10) try: for word, value in seg_txt_2_dict(keywords).items(): if word != "none": doc.add_term(word, 10) else: pass except: pass database.add_document(doc) add_num = add_num + 1 if (Globals.DEBUG_SWITCH): print("App:", doc.get_data(), " ", "terms:", end=' ') for itr in doc.termlist(): if (Globals.DEBUG_SWITCH): print(itr.term, end=' ') xapiandb_update = "Yes" if (Globals.DEBUG_SWITCH): print(" ") try: if xapiandb_update == "Yes": now = time.strftime('%Y-%m-%dT%H:%M:%S', time.localtime()) doc_for_xapiandb_version.add_value(2, now) database.replace_document(docid_for_xapiandb_version, doc_for_xapiandb_version) database.commit() if (Globals.DEBUG_SWITCH): print( "Xapiandb has updated . %d app modified, %d app add. Tatal: %d app updated" % (modified_num, add_num, len(reslist))) except: if (Globals.DEBUG_SWITCH): print( "The xapian database (/home/ice_bird/.cache/uksc/xapiandb) is crashed,please remove it and install a new one!" ) if (Globals.DEBUG_SWITCH): print("update uksc xapiandb over") else: appinfo_query = xapian.Query(kwargs["pkgname"]) enquire.set_query(appinfo_query) matches = enquire.get_mset(0, DB.get_doccount()) for re in matches: doc_for_appinfo = re.document doc_data = doc_for_appinfo.get_data() if kwargs["pkgname"] == doc_data: return doc = xapian.Document() doc.set_data(kwargs["pkgname"]) doc.add_term(kwargs["pkgname"], 10) if (Globals.DEBUG_SWITCH): print("debfile path:", kwargs["path"]) deb = DebFile(kwargs["path"]) terms = kwargs["pkgname"] try: terms = terms + " " + deb.description except: if (Globals.DEBUG_SWITCH): print("Failed to get app description") indexer.set_document(doc) indexer.index_text(terms) database.add_document(doc) database.commit() if (Globals.DEBUG_SWITCH): print("update xapiandb over: ", kwargs["pkgname"], "terms:", end=' ') for itr in doc.termlist(): if (Globals.DEBUG_SWITCH): print(itr.term, end=' ') if (Globals.DEBUG_SWITCH): print(" ")
class SilentProcess(multiprocessing.Process): def __init__(self, squeue): super(SilentProcess, self).__init__() multiprocessing.Process.__init__(self) self.daemon = True self.squeue = squeue self.destFile = os.path.join(UKSC_CACHE_DIR,"uksc.db") self.connect = sqlite3.connect(self.destFile, check_same_thread=False) self.cursor = self.connect.cursor() self.premoter = PistonRemoter(service_root=UBUNTUKYLIN_SERVER) def run(self): while True: workqueuelen = self.squeue.qsize() # print "silent worklist size : ", str(workqueuelen) if workqueuelen == 0: time.sleep(1) continue item = self.squeue.get_nowait() print "silent process get one workitem : ", item.funcname if item.funcname == "get_all_ratings": self.get_all_ratings() elif item.funcname == "submit_pingback_main": self.submit_pingback_main() elif item.funcname == "submit_pingback_app": self.submit_pingback_app(item.kwargs) elif item.funcname == "get_all_categories": self.get_all_categories() elif item.funcname == "get_all_rank_and_recommend": self.get_all_rank_and_recommend() elif item.funcname == "get_newer_application_info": self.get_newer_application_info() # update rating_avg and rating_total in cache db from server def get_all_ratings(self): reslist = self.premoter.get_all_ratings() for rating in reslist: app_name = rating['app_name'] rating_avg = str(rating['rating_avg']) rating_total = str(rating['rating_total']) sql = "update application set rating_total=?,rating_avg=? where app_name=?" self.cursor.execute(sql, (rating_total,rating_avg,app_name)) self.connect.commit() print "all ratings and rating_total update over : ",len(reslist) # submit pingback-main to server def submit_pingback_main(self): machine = get_machine_id() distro = get_distro_info()[0] version_os = get_distro_info()[1] version_uksc = get_uksc_version() res = self.premoter.submit_pingback_main(machine, distro, version_os, version_uksc) return res # submit pingback-app to server def submit_pingback_app(self, kwargs): app_name = kwargs["app_name"] isrcm = kwargs["isrcm"] machine = get_machine_id() res = self.premoter.submit_pingback_app(app_name, machine, isrcm) return res # get all categories data from server def get_all_categories(self): reslist = self.premoter.get_all_categories() for category in reslist: cid = category['id'] name = category['name'] display_name = category['display_name'] priority = category['priority'] sql = "select count(*) from category where id=?" self.cursor.execute(sql, (cid,)) res = self.cursor.fetchall() isexist = '' for item in res: isexist = item[0] if(isexist == 1): # id exist, update sql = "update category set name=?,display_name=?,priority=? where id=?" self.cursor.execute(sql, (name,display_name,priority,cid)) else: # id not exist, insert sql = "insert into category(id,name,display_name,priority,visible) values(?,?,?,?,1)" self.cursor.execute(sql, (cid,name,display_name,priority)) self.connect.commit() print "all categories update over : ",len(reslist) # get all rank and recommend data from server def get_all_rank_and_recommend(self): reslist = self.premoter.get_all_rank_and_recommend() sql = "delete from rank" self.cursor.execute(sql) for rank in reslist: rid = rank['id'] aid = rank['aid']['id'] rank_rating = rank['rank_rating'] rank_download = rank['rank_download'] rank_recommend = rank['rank_recommend'] rank_pointout = rank['rank_pointout'] sql = "insert into rank(id,aid_id,rank_rating,rank_download,rank_recommend,rank_pointout) values(?,?,?,?,?,?)" self.cursor.execute(sql, (rid,aid,rank_rating,rank_download,rank_recommend,rank_pointout)) self.connect.commit() print "all rank and recommend update over : ",len(reslist) # get newer application info from server def get_newer_application_info(self): # get application info last update date last_update_date = '' self.cursor.execute("select value from dict where key='appinfo_updatetime'") res = self.cursor.fetchall() for item in res: last_update_date = item[0] reslist = self.premoter.get_newer_application_info(last_update_date) # update application info to cache db for app in reslist: aid = app['id'] app_name = app['app_name'] display_name = app['display_name'] display_name_cn = app['display_name_cn'] categories = app['categories'] summary = app['summary'] description = app['description'] command = app['command'] rating_avg = app['rating_avg'] rating_total = app['rating_total'] review_total = app['review_total'] download_total = app['download_total'] sql = "select count(*) from application where id=?" self.cursor.execute(sql, (aid,)) res = self.cursor.fetchall() isexist = '' for item in res: isexist = item[0] if(isexist == 1): # id exist, update sql = "update application set app_name=?,display_name=?,display_name_cn=?,categories=?,summary=?,description=?,command=?,rating_avg=?,rating_total=?,review_total=?,download_total=? where id=?" self.cursor.execute(sql, (app_name,display_name,display_name_cn,categories,summary,description,command,rating_avg,rating_total,review_total,download_total,aid)) else: # id not exist, insert sql = "insert into application(id,app_name,display_name,display_name_cn,categories,summary,description,command,rating_avg,rating_total,review_total,download_total) values(?,?,?,?,?,?,?,?,?,?,?,?)" self.cursor.execute(sql, (aid,app_name,display_name,display_name_cn,categories,summary,description,command,rating_avg,rating_total,review_total,download_total)) # set application info last update date nowdate = time.strftime('%Y-%m-%d',time.localtime()) self.cursor.execute("update dict set value=? where key=?", (nowdate,'appinfo_updatetime')) self.connect.commit() print "all newer application info update over : ",len(reslist)
class Database: def __init__(self): self.updatecount = 0 srcFile = os.path.join(UBUNTUKYLIN_DATA_PATH,"uksc.db") destFile = os.path.join(UKSC_CACHE_DIR,"uksc.db") # no cache file, copy if not os.path.exists(destFile): if not os.path.exists(srcFile): print "error with db file" return open(destFile, "wb").write(open(srcFile, "rb").read()) self.connect = sqlite3.connect(destFile, check_same_thread=False) self.cursor = self.connect.cursor() self.cat_list = [] # cache file need update, copy if self.is_cachedb_need_update(): open(destFile, "wb").write(open(srcFile, "rb").read()) # piston remoter to ukscs self.premoter = PistonRemoter(service_root=UBUNTUKYLIN_SERVER) def query_categories(self): self.cursor.execute("select * from category") res = self.cursor.fetchall() # print "query_categories:",len(res),res return res def query_category_apps(self, cate_name): al = '' sql = "select id from category where name='%s'" self.cursor.execute(sql % cate_name) res = self.cursor.fetchall() cateid = '' for i in res: cateid = i[0] sql = "select id,categories from application" self.cursor.execute(sql) res = self.cursor.fetchall() for i in res: aid = i[0] cstring = i[1] cs = cstring.split(',') for c in cs: if(int(cateid) == int(c)): al += str(aid) al += ',' al = al[:-1] sql = "select app_name,display_name_cn from application where id in (%s) order by rating_total DESC" self.cursor.execute(sql % al) res = self.cursor.fetchall() # for a in res: # print a[0]," ",a[1] return res #return as (display_name, summary, description, rating_average,rating_total,review_total,download_total) def query_application(self,pkgname): self.cursor.execute(QUERY_APP % (pkgname)) res = self.cursor.fetchall() # print "query_application:",pkgname,len(res),res if len(res)==0: return [] else: return res[0] #return as (display_name, app_name) def query_applications(self): self.cursor.execute(QUERY_APPS) res = self.cursor.fetchall() # print "query_application:",pkgname,len(res),res if len(res)==0: return [] else: return res def update_app_rnr(self,pkgname,rating_average,rating_total,review_total,download_total=0): print "update_app_rnr:",self.updatecount,pkgname,rating_average,rating_total,review_total,download_total self.cursor.execute(UPDATE_APP_RNR % (rating_average,rating_total,review_total,download_total,pkgname)) self.connect.commit() #res = self.cursor.fetchall() #print "update_app_rnr:",len(res),res self.updatecount += 1 return True #---------------------------------0.3---------------------------------- expiredict = {} # check the ~/.cache/uksc/uksc.db version, and copy /usr/share/u../data/uksc.db to replace it def is_cachedb_need_update(self): srcFile = os.path.join(UBUNTUKYLIN_DATA_PATH,"uksc.db") connectsrc = sqlite3.connect(srcFile, check_same_thread=False) cursorsrc = connectsrc.cursor() self.cursor.execute("select count(*) from sqlite_master where type='table' and name='dict'") res = self.cursor.fetchall() dictcount = '' for item in res: dictcount = item[0] if(dictcount == 0): return True self.cursor.execute("select value from dict where key='dbversion'") res = self.cursor.fetchall() olddbversion = '' for item in res: olddbversion = int(item[0]) cursorsrc.execute("select value from dict where key='dbversion'") res = cursorsrc.fetchall() newdbversion = '' for item in res: newdbversion = int(item[0]) if(newdbversion > olddbversion): return True return False def get_pagecount_by_pkgname(self, package_name): self.cursor.execute("select review_total from application where app_name=?", (package_name,)) res = self.cursor.fetchall() for item in res: review_total = item[0] return review_total / 10 + 1 def get_review_by_pkgname(self, package_name, page): # get application id self.cursor.execute("select id from application where app_name=?", (package_name,)) res = self.cursor.fetchall() aid = '' for item in res: aid = str(item[0]) # get review count self.cursor.execute("select count(*) from review where aid_id=?", (aid,)) res = self.cursor.fetchall() count = '' for item in res: count = item[0] if(page == 1): # empty cache, download page 1 if(count == 0): reviews = self.premoter.get_reviews(package_name, 0, 10) review_total = '' for review in reviews: id = str(review.id) review_total = review.aid['review_total'] user_display = review.user_display content = review.content date = str(review.date) date = date.replace('T',' ') date = date.replace('Z','') self.cursor.execute("insert into review values(?,?,'',?,?,?,'zh_CN','',0,0)", (id,aid,content,user_display,date)) if(review_total != ''): self.cursor.execute("update application set review_total=? where id=?", (review_total,aid)) self.connect.commit() # normal init, check and download newest reviews elif(count != 0): # get newest review's id from local cache self.cursor.execute("select id from review where aid_id=? order by date DESC limit 0,1", (aid,)) res = self.cursor.fetchall() id = '' for item in res: id = item[0] # find newest reviews from server startpage = 0 loop = True while loop: reviews = self.premoter.get_reviews(package_name, startpage, 10) review_total = '' for review in reviews: rid = review.id review_total = review.aid['review_total'] user_display = review.user_display content = review.content date = str(review.date) date = date.replace('T',' ') date = date.replace('Z','') # download if(rid != id): self.cursor.execute("insert or ignore into review values(?,?,'',?,?,?,'zh_CN','',0,0)", (rid,aid,content,user_display,date)) # end download else: # stop 'while' loop = False # break 'for' break if(review_total != ''): self.cursor.execute("update application set review_total=? where id=?", (review_total,aid)) startpage += 1 self.connect.commit() else: # review not enough, download if(count < page * 10): start = (page - 1) * 10 reviews = self.premoter.get_reviews(package_name, start, 10) review_total = '' for review in reviews: id = str(review.id) review_total = review.aid['review_total'] user_display = review.user_display content = review.content date = str(review.date) date = date.replace('T',' ') date = date.replace('Z','') # ignore the same review by id self.cursor.execute("insert or ignore into review values(?,?,'',?,?,?,'zh_CN','',0,0)", (id,aid,content,user_display,date)) if(review_total != ''): self.cursor.execute("update application set review_total=? where id=?", (review_total,aid)) self.connect.commit() # all download check over, return reviews to show limit = (page - 1) * 10 self.cursor.execute("select id, aid_id, content, user_display, date, language, up_total, down_total from review where aid_id=? order by date DESC limit ?,10", (aid,limit)) res = self.cursor.fetchall() reviews = [] for item in res: review = Review(package_name) review.id = item[0] review.aid = item[1] review.content = item[2] review.user_display = item[3] review.date = item[4] review.language = item[5] review.up_total = item[6] review.down_total = item[7] reviews.append(review) return reviews def get_pointout_is_show(self): self.cursor.execute("select value from dict where key=?", ('pointout',)) res = self.cursor.fetchall() for item in res: isshow = item[0] return isshow def set_pointout_is_show(self, flag): value = '' if(flag == True): value = 'True' else: value = 'False' self.cursor.execute("update dict set value=? where key='pointout'", (value,)) self.connect.commit() def get_pointout_apps(self): self.cursor.execute("select app_name,rank_pointout from rank,application where rank_pointout!=0 and rank.aid_id=application.id order by rank_pointout") res = self.cursor.fetchall() pointouts = [] for item in res: app_name = item[0] rank_pointout = item[1] pointouts.append((app_name, rank_pointout)) return pointouts def get_recommend_apps(self): self.cursor.execute("select app_name,rank_recommend from rank,application where rank_recommend!=0 and rank.aid_id=application.id order by rank_recommend") res = self.cursor.fetchall() recommends = [] for item in res: app_name = item[0] rank_recommend = item[1] recommends.append((app_name, rank_recommend)) return recommends def get_ratingrank_apps(self): self.cursor.execute("select app_name,rank_rating from rank,application where rank_rating!=0 and rank.aid_id=application.id order by rank_rating") res = self.cursor.fetchall() ratingranks = [] for item in res: app_name = item[0] rank_rating = item[1] ratingranks.append((app_name, rank_rating)) return ratingranks #------------add by kobe for windows replace------------ def search_name_and_categories_record(self): self.cursor.execute(QUERY_NAME_CATEGORIES) res = self.cursor.fetchall() if len(res) == 0: return [] else: return res #------------add by kobe for windows replace------------ def search_app_display_info(self, categories): self.cursor.execute(QUERY_APP_ACCORD_CATEGORIES % (categories)) res = self.cursor.fetchall() if len(res) == 0: return [] else: return res #------------add by kobe for windows replace------------ def update_exists_data(self, exists, id): self.cursor.execute(UPDATE_EXISTS % (exists, id)) self.connect.commit()