def save_product(self, product, category, store_id): try: if (not product) or (not category): return None result = None # img = product.get('img') # 如果产品图片少于8张则填充None # if len(img) < 8: # img += [None] * (8 - len(img)) # 通过产品链接查询该产品是否已经存在 QUERY_PRODUCT_ID = "SELECT product_id FROM product WHERE product_href='" + product.get( 'href') + "'" INSERT_PRODUCT = "INSERT INTO product(product_name,price,quantity_uom_id,currency_uom_id,product_href,product_img_0,product_img_1,product_img_2,product_img_3,product_img_4,product_img_5,product_img_6,product_img_7,description,long_description,category_2nd_id,store_id) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING product_id" result = DBUtil().query_one(QUERY_PRODUCT_ID) if result: return str(result) result = DBUtil().insert( INSERT_PRODUCT, (product.get('title'), product.get('price'), product.get('unit'), product.get('currency'), product.get('href'), None, None, None, None, None, None, None, None, None, None, category, store_id)) except BaseException: self.logger.error('保存产品信息出错:' + traceback.format_exc()) return str(result)
def __init__(self): super(MainServer, self).__init__(socket.AF_INET, socket.SOCK_STREAM) # Getting a connection to the DB. self.db = DBUtil(DB_CONFIG) # Creating the necessary tables. for more information see the DDLs above... self.create_tables() self.socket_to_address_dict = {} self.socket_to_username_dict = {} # Creating dictionary that maps from client to the latest captcha that has been send to him. self.last_captcha_text = {} self.captcha_image_generator = ImageCaptcha(fonts=FONTS) self.captcha_audio_generator = AudioCaptcha(voicedir=VOICES_DIR) # Listening to the required IP and PORT. self.bind((BIND_IP, BIND_PORT)) # Limiting the number of clients to BACKLOG. self.listen(BACKLOG) # starting a new thread that accepts new clients Thread(target=self.accept_clients).start()
def __init__(self): db = mongodb DBUtil.__init__(self, db) self.collection = db.Reports # indexes self.collection.create_index([('name', ASCENDING)], unique=True)
def producepicture(spm, startdate, enddate): conn = DBUtil() tablename = conn.tablename print("现在处理的spm:", spm) sql = "SELECT slideend_x,slideend_y,entity_x,entity_y,entity_width,entity_height FROM " \ +tablename+" where spm=%s and dt>=%s and dt<=%s and touch_type=2 order by pos limit 0,100000; " args = (spm, startdate, enddate) results = conn.executesearch(sql, args) conf = config() processim = np.zeros([conf.heightborder, conf.widthborder], dtype=int) #高度和宽度 count = 0 for data in results: if judegzero(data[4], data[5]) != 0: x = int((data[0] - data[2]) / data[4] * conf.widthborder) # 鼠标点击位置减去容器框位置除以容器框的宽度 y = int((data[1] - data[3]) / data[5] * conf.heightborder) #360 120 另外一组是 if judgeoutborder(x, y): count = count + 1 processim[y, x] = processim[y, x] + 1 if count % 1000 == 0: print("处理数据进度:", count) print(str(data)) maxcount = np.max(processim) print("最大点击次数为:", maxcount) processim = processim * 255 / maxcount new_path = "imgs/" + spm + startdate + enddate + ".png" print("总点击次数为:", count) if (count >= 1000): csv_operator.saveexecl(spm, maxcount, count) #scipy.misc.imsave(new_path, processim) cv2.imwrite(new_path, processim)
def __init__(self, urls): self.__rootPath = "https://www.cnblogs.com" self.__allUrl = set() # 已经访问过的URL self.__resultUrl = set() # 结果集 self.__newUrl = set(urls) # 新添加的URL self.__dbUtil = DBUtil() self.__resultL = 0 pass
def __init__(self, db): ''' General description: This function initializes the database variables and \ index to refer in functions. ''' DBUtil.__init__(self, db) self.collection = db.Emails
def __init__(self, urls): self.__rootPath = "http://blog.sina.com.cn" self.__allUrl = {"http://blog.sina.com.cn/showpic.html"} # 已经访问过的URL self.__resultUrl = set() # 结果集 self.__newUrl = set(urls) # 新添加的URL self.__dbUtil = DBUtil() self.__resultL = 0 pass
def __init__(self, db): ''' General description: This function initializes the database variables and index to refer in functions. ''' DBUtil.__init__(self, db) self.collection = db.ContributionGitPushLogs
def __init__(self, db): ''' General description: This function initializes the database variables and \ index to refer in functions. ''' DBUtil.__init__(self, db) self.collection = db.Sync self.system_details_db = SystemDetails.SystemDetails(db)
def __init__(self, db): ''' General description: This function initializes the database variables and \ index to refer in functions. ''' DBUtil.__init__(self, db) self.collection = db.Config self.passHelper = PasswordHelper.PasswordHelper(key)
def __init__(self, db): ''' General description: This function initializes the database variables and \ index to refer in functions. ''' DBUtil.__init__(self, db) self.collection = db.Plugins # indexes self.collection.create_index([('name', ASCENDING)], unique=True)
def spmlist(startdate, enddate): conn = DBUtil() conf = config() tablename = conf.tablename sql = "SELECT distinct spm FROM " \ +tablename+" where dt>=%s and dt<=%s and touch_type=2 ; " args = (startdate, enddate) results = conn.executesearch(sql, args) return results
def __init__(self, db): ''' General description: This function initializes the database variables and \ index to refer in functions. ''' DBUtil.__init__(self, db) self.collection = db.MachineGroups self.machineDB = Machine.Machine(db) # indexes self.collection.create_index([('group_name', ASCENDING)], unique=True)
def __init__(self, db): ''' General description : This function initializes the database variables and \ index to refer in functions. ''' DBUtil.__init__(self, db) self.collection = db.DeploymentFields # indexes self.collection.create_index( [('parent_entity_id', ASCENDING)], unique=True)
def query_category(self): category_2nd_id = '9999' result = [] LAST_CATEGORY_2ND = 'SELECT category_2nd_id FROM product ORDER BY product_id DESC LIMIT 1' LOAD_CATEGORY = 'SELECT category_2nd_id,category_href FROM category_2nd WHERE category_2nd_id>' try: category_2nd_id = DBUtil().query_one(LAST_CATEGORY_2ND) result = DBUtil().query_list(LOAD_CATEGORY + category_2nd_id) except BaseException: self.logger.error('查询分类出错:' + traceback.format_exc()) return result
def Save_To_Database(record): for onerecord in record: picturelocation = rootpath + img_path + "/" + onerecord[0] spm = onerecord[0][:-20] likelihood = onerecord[1] conn = DBUtil() tablename = conf.tablename sql = "REPLACE INTO " + tablename + " SET spm = %s, Normal_possibility = %s, location = %s;" #注意这行语句只能在spm为主码时使用 args = (spm, str(likelihood), picturelocation) conn.executesearch(sql, args) print("save to database over!")
def __init__(self, db): DBUtil.__init__(self, db) self.collection = db.Machine self.passHelper = PasswordHelper.PasswordHelper(key) self.userdb = Users.Users(db) self.roleDb = Role.Role(db) self.tagDB = Tags.Tags() # indexes self.collection.create_index([('username', ASCENDING), ('host', ASCENDING)], unique=True)
def __init__(self): ''' General description: This function initializes the database variables and \ index to refer in functions. ''' db = mongodb DBUtil.__init__(self, db) self.collection = db.DeploymentUnitApprovalStatus # indexes self.collection.create_index([('name', ASCENDING)], unique=True)
def __init__(self): ''' General description: This function initializes the database variables and \ index to refer in functions. ''' db = mongodb DBUtil.__init__(self, db) self.collection = db.Repository self.passHelper = PasswordHelper.PasswordHelper(key) # indexes self.collection.create_index([('name', ASCENDING)], unique=True)
def __init__(self, db): ''' General description: This function initializes the database variables and \ index to refer in functions. ''' DBUtil.__init__(self, db) self.Collection = db.ToolInstallation # indexes self.Collection.create_index([('version_id', ASCENDING), ('machine_id', ASCENDING)], unique=True)
class BlogRepyilt: def __init__(self, urls): self.__rootPath = "http://blog.sina.com.cn" self.__allUrl = {"http://blog.sina.com.cn/showpic.html"} # 已经访问过的URL self.__resultUrl = set() # 结果集 self.__newUrl = set(urls) # 新添加的URL self.__dbUtil = DBUtil() self.__resultL = 0 pass def getUrl(self): urls = self.__newUrl.copy() # 拷贝一份,一边遍历一边删除 try: for url in urls: if url.find("showpic") == -1: self.__allUrl.add(url) # 已经访问了 self.__newUrl.remove(url) # 删除已经访问过的 html = None try: html = requests.get(url) except Exception as e: print(e) continue bs = BeautifulSoup(html.text, features="html.parser") hrefs = bs.find_all("a") for href in hrefs: href = href.get("href") if href != None and href != "" and href[0] == "/": url = self.__rootPath + str(href) else: url = href if href != None and self.__allUrl.__contains__( url) == False: # 是否已经访问 if url.find("blog") >= 0 and url.find( "sina") >= 0: # 是否是符合规则的URL self.__newUrl.add(url) # 符合规则添加到新的URL中 else: self.__allUrl.add(url) # 不符合规则直接扔掉 if self.__resultUrl.__contains__( url ) == False and re.match( "http://blog\.sina\.com\.cn/s/.*.html\?tj=1", url) is not None: self.__resultUrl.add(url) self.__dbUtil.insertTbUrl(url) self.__resultL += 1 print("目前url条数:" + str(self.__resultL)) except Exception as e: print(e) self.getUrl() pass
def __init__(self, db): ''' General description: This function initializes the database variables and \ index to refer in functions. ''' DBUtil.__init__(self, db) self.collection = db.DeploymentRequest self.machineDB = Machine.Machine(db) self.versionsDB = Versions.Versions(db) self.toolDB = Tool.Tool(db) self.passHelper = PasswordHelper.PasswordHelper(key) self.statedb = State.State(db)
def __init__(self, db): ''' General description: This function initializes the database variables and \ index to refer in functions. ''' DBUtil.__init__(self, db) self.collection = db.SystemDetails self.configDb = Config.Config(mongodb) # indexes # self.collection.create_index([('version_id', ASCENDING)], unique=False) self.collection.create_index([('ip', ASCENDING)], unique=True) self.collection.create_index([('hostname', ASCENDING)], unique=True)
def __init__(self, db): ''' General description: This function initializes the database variables and \ index to refer in functions. ''' DBUtil.__init__(self, db) self.collection = db.Tool self.versionsDB = Versions.Versions(db) self.tagDB = Tags.Tags() # indexes self.collection.create_index([('name', ASCENDING)], unique=True) self.collection.create_index([('tag', ASCENDING)], unique=False)
def query_store(self): result = [] # store_id = '9999' # LAST_STORE_ID = 'SELECT store_id FROM store WHERE linkman IS NOT NULL ORDER BY store_id DESC LIMIT 1' LAST_STORE_ID = "SELECT store_id FROM store WHERE store_homepage='http://shanghaifhm.b2b.hc360.com'" # LOAD_STORE = "SELECT store_id,store_homepage FROM store WHERE linkman IS NULL" LOAD_STORE = 'SELECT store_id,store_homepage FROM store WHERE linkman IS NULL AND store_id>' try: store_id = DBUtil().query_one(LAST_STORE_ID) result = DBUtil().query_list(LOAD_STORE + store_id) # result = DBUtil().query_list(LOAD_STORE) except BaseException: self.logger.error('查询现有商家出错:' + traceback.format_exc()) return result
def __init__(self, db): ''' General description : This function initializes the database variables and \ index to refer in functions. ''' DBUtil.__init__(self, db) self.collection = db.DeploymentRequestGroup self.deploymentRequestDb = DeploymentRequest.DeploymentRequest(db) self.machineDB = Machine.Machine(db) self.versionsDB = Versions.Versions(db) self.toolDB = Tool.Tool(db) self.deploymentUnitDB = DeploymentUnit.DeploymentUnit() self.deploymentRequestDbCollection = db.DeploymentRequest self.statedb = State.State(db)
def __init__(self, db): ''' General description: This function initializes the database variables and \ index to refer in functions. ''' DBUtil.__init__(self, db) self.collection = db.UserFavoriteMachine self.machineTypeDb = MachineType.MachineType(db) self.machineDB = Machine.Machine(db) self.userDB = Users.Users(db) # indexes self.collection.create_index([('user_id', ASCENDING), ('machine_id', ASCENDING)], unique=True)
def __init__(self, db): ''' General description: This function initializes the database variables and \ index to refer in functions. ''' DBUtil.__init__(self, db) self.collection = db.PermissionGroup self.permissiondb = Permissions.Permissions(db) self.routesb = Routes.Routes(db) # indexes self.collection.create_index( [('groupname', ASCENDING)], unique=True, )
def __init__(self, db): ''' General description: This function initializes the database variables and \ index to refer in functions. ''' DBUtil.__init__(self, db) self.collection = db.State self.deploymentunitapprovalstatusdb = DeploymentUnitApprovalStatus.DeploymentUnitApprovalStatus( ) self.DeploymentFieldsdb = DeploymentFields.DeploymentFields(db) self.buildDb = Build.Build() # indexes self.collection.create_index([('name', ASCENDING), ('parent_entity_id', ASCENDING)], unique=True)
def query_products(self): result = [] LOAD_PRODUCT = 'SELECT product_id,product_href FROM product WHERE product_img_0 IS NOT NULL' try: result = DBUtil().query_list(LOAD_PRODUCT) except BaseException: self.logger.error('查询所有产品出错:' + traceback.format_exc()) return result