Esempio n. 1
0
 def save_product(self, product, category, store_id):
     try:
         if (not product) or (not category):
             return None
         result = None
         # img = product.get('img')
         # 如果产品图片少于8张则填充None
         # if len(img) < 8:
         #     img += [None] * (8 - len(img))
         # 通过产品链接查询该产品是否已经存在
         QUERY_PRODUCT_ID = "SELECT product_id FROM product WHERE product_href='" + product.get(
             'href') + "'"
         INSERT_PRODUCT = "INSERT INTO product(product_name,price,quantity_uom_id,currency_uom_id,product_href,product_img_0,product_img_1,product_img_2,product_img_3,product_img_4,product_img_5,product_img_6,product_img_7,description,long_description,category_2nd_id,store_id) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING product_id"
         result = DBUtil().query_one(QUERY_PRODUCT_ID)
         if result:
             return str(result)
         result = DBUtil().insert(
             INSERT_PRODUCT,
             (product.get('title'), product.get('price'),
              product.get('unit'), product.get('currency'),
              product.get('href'), None, None, None, None, None, None, None,
              None, None, None, category, store_id))
     except BaseException:
         self.logger.error('保存产品信息出错:' + traceback.format_exc())
     return str(result)
    def __init__(self):
        super(MainServer, self).__init__(socket.AF_INET, socket.SOCK_STREAM)

        # Getting a connection to the DB.
        self.db = DBUtil(DB_CONFIG)

        # Creating the necessary tables. for more information see the DDLs above...
        self.create_tables()

        self.socket_to_address_dict = {}

        self.socket_to_username_dict = {}

        # Creating dictionary that maps from client to the latest captcha that has been send to him.
        self.last_captcha_text = {}

        self.captcha_image_generator = ImageCaptcha(fonts=FONTS)

        self.captcha_audio_generator = AudioCaptcha(voicedir=VOICES_DIR)

        # Listening to the required IP and PORT.
        self.bind((BIND_IP, BIND_PORT))

        # Limiting the number of clients to BACKLOG.
        self.listen(BACKLOG)

        # starting a new thread that accepts new clients
        Thread(target=self.accept_clients).start()
Esempio n. 3
0
    def __init__(self):
        db = mongodb
        DBUtil.__init__(self, db)
        self.collection = db.Reports

        # indexes
        self.collection.create_index([('name', ASCENDING)], unique=True)
Esempio n. 4
0
def producepicture(spm, startdate, enddate):
    conn = DBUtil()
    tablename = conn.tablename
    print("现在处理的spm:", spm)
    sql = "SELECT slideend_x,slideend_y,entity_x,entity_y,entity_width,entity_height  FROM " \
          +tablename+" where spm=%s and dt>=%s and dt<=%s and touch_type=2 order by pos limit 0,100000; "
    args = (spm, startdate, enddate)
    results = conn.executesearch(sql, args)

    conf = config()
    processim = np.zeros([conf.heightborder, conf.widthborder],
                         dtype=int)  #高度和宽度
    count = 0
    for data in results:
        if judegzero(data[4], data[5]) != 0:
            x = int((data[0] - data[2]) / data[4] *
                    conf.widthborder)  # 鼠标点击位置减去容器框位置除以容器框的宽度
            y = int((data[1] - data[3]) / data[5] *
                    conf.heightborder)  #360 120 另外一组是
            if judgeoutborder(x, y):
                count = count + 1
                processim[y, x] = processim[y, x] + 1
                if count % 1000 == 0:
                    print("处理数据进度:", count)
                    print(str(data))

    maxcount = np.max(processim)
    print("最大点击次数为:", maxcount)
    processim = processim * 255 / maxcount
    new_path = "imgs/" + spm + startdate + enddate + ".png"
    print("总点击次数为:", count)
    if (count >= 1000):
        csv_operator.saveexecl(spm, maxcount, count)
        #scipy.misc.imsave(new_path, processim)
        cv2.imwrite(new_path, processim)
Esempio n. 5
0
 def __init__(self, urls):
     self.__rootPath = "https://www.cnblogs.com"
     self.__allUrl = set()  # 已经访问过的URL
     self.__resultUrl = set()  # 结果集
     self.__newUrl = set(urls)  # 新添加的URL
     self.__dbUtil = DBUtil()
     self.__resultL = 0
     pass
Esempio n. 6
0
 def __init__(self, db):
     '''
        General description:
         This function initializes the database variables and \
         index to refer in functions.
     '''
     DBUtil.__init__(self, db)
     self.collection = db.Emails
Esempio n. 7
0
 def __init__(self, urls):
     self.__rootPath = "http://blog.sina.com.cn"
     self.__allUrl = {"http://blog.sina.com.cn/showpic.html"}  # 已经访问过的URL
     self.__resultUrl = set()  # 结果集
     self.__newUrl = set(urls)  # 新添加的URL
     self.__dbUtil = DBUtil()
     self.__resultL = 0
     pass
Esempio n. 8
0
 def __init__(self, db):
     '''                                                            
       General description:                                           
       This function initializes the database variables and           
       index to refer in functions.                                   
     '''
     DBUtil.__init__(self, db)
     self.collection = db.ContributionGitPushLogs
Esempio n. 9
0
 def __init__(self, db):
     '''
         General description:
         This function initializes the database variables and \
         index to refer in functions.
     '''
     DBUtil.__init__(self, db)
     self.collection = db.Sync
     self.system_details_db = SystemDetails.SystemDetails(db)
Esempio n. 10
0
 def __init__(self, db):
     '''
        General description:
         This function initializes the database variables and \
         index to refer in functions.
     '''
     DBUtil.__init__(self, db)
     self.collection = db.Config
     self.passHelper = PasswordHelper.PasswordHelper(key)
Esempio n. 11
0
 def __init__(self, db):
     '''
         General description:
         This function initializes the database variables and \
         index to refer in functions.
     '''
     DBUtil.__init__(self, db)
     self.collection = db.Plugins
     # indexes
     self.collection.create_index([('name', ASCENDING)], unique=True)
Esempio n. 12
0
def spmlist(startdate, enddate):
    conn = DBUtil()
    conf = config()
    tablename = conf.tablename
    sql = "SELECT distinct spm  FROM " \
          +tablename+" where  dt>=%s and dt<=%s and touch_type=2  ; "
    args = (startdate, enddate)
    results = conn.executesearch(sql, args)

    return results
Esempio n. 13
0
 def __init__(self, db):
     '''
        General description:
        This function initializes the database variables and \
        index to refer in functions.
     '''
     DBUtil.__init__(self, db)
     self.collection = db.MachineGroups
     self.machineDB = Machine.Machine(db)
     # indexes
     self.collection.create_index([('group_name', ASCENDING)], unique=True)
Esempio n. 14
0
 def __init__(self, db):
     '''                                                                
      General description :                                           
      This function initializes the database variables and \          
      index to refer in functions.                                    
     '''
     DBUtil.__init__(self, db)
     self.collection = db.DeploymentFields
     # indexes
     self.collection.create_index(
         [('parent_entity_id', ASCENDING)], unique=True)
Esempio n. 15
0
 def query_category(self):
     category_2nd_id = '9999'
     result = []
     LAST_CATEGORY_2ND = 'SELECT category_2nd_id FROM product ORDER BY product_id DESC LIMIT 1'
     LOAD_CATEGORY = 'SELECT category_2nd_id,category_href FROM category_2nd WHERE category_2nd_id>'
     try:
         category_2nd_id = DBUtil().query_one(LAST_CATEGORY_2ND)
         result = DBUtil().query_list(LOAD_CATEGORY + category_2nd_id)
     except BaseException:
         self.logger.error('查询分类出错:' + traceback.format_exc())
     return result
Esempio n. 16
0
def Save_To_Database(record):
    for onerecord in record:
        picturelocation = rootpath + img_path + "/" + onerecord[0]
        spm = onerecord[0][:-20]
        likelihood = onerecord[1]
        conn = DBUtil()
        tablename = conf.tablename
        sql = "REPLACE INTO " + tablename + " SET  spm = %s, Normal_possibility = %s, location = %s;"  #注意这行语句只能在spm为主码时使用
        args = (spm, str(likelihood), picturelocation)
        conn.executesearch(sql, args)
    print("save to database over!")
Esempio n. 17
0
 def __init__(self, db):
     DBUtil.__init__(self, db)
     self.collection = db.Machine
     self.passHelper = PasswordHelper.PasswordHelper(key)
     self.userdb = Users.Users(db)
     self.roleDb = Role.Role(db)
     self.tagDB = Tags.Tags()
     # indexes
     self.collection.create_index([('username', ASCENDING),
                                   ('host', ASCENDING)],
                                  unique=True)
    def __init__(self):
        '''
           General description:
            This function initializes the database variables and \
            index to refer in functions.
        '''
        db = mongodb
        DBUtil.__init__(self, db)
        self.collection = db.DeploymentUnitApprovalStatus

        # indexes
        self.collection.create_index([('name', ASCENDING)], unique=True)
Esempio n. 19
0
 def __init__(self):
     '''
        General description:
         This function initializes the database variables and \
         index to refer in functions.
     '''
     db = mongodb
     DBUtil.__init__(self, db)
     self.collection = db.Repository
     self.passHelper = PasswordHelper.PasswordHelper(key)
     # indexes
     self.collection.create_index([('name', ASCENDING)], unique=True)
Esempio n. 20
0
 def __init__(self, db):
     '''
     General description:
     This function initializes the database variables and \
     index to refer in functions.
     '''
     DBUtil.__init__(self, db)
     self.Collection = db.ToolInstallation
     # indexes
     self.Collection.create_index([('version_id', ASCENDING),
                                   ('machine_id', ASCENDING)],
                                  unique=True)
Esempio n. 21
0
class BlogRepyilt:
    def __init__(self, urls):
        self.__rootPath = "http://blog.sina.com.cn"
        self.__allUrl = {"http://blog.sina.com.cn/showpic.html"}  # 已经访问过的URL
        self.__resultUrl = set()  # 结果集
        self.__newUrl = set(urls)  # 新添加的URL
        self.__dbUtil = DBUtil()
        self.__resultL = 0
        pass

    def getUrl(self):
        urls = self.__newUrl.copy()  # 拷贝一份,一边遍历一边删除
        try:
            for url in urls:
                if url.find("showpic") == -1:
                    self.__allUrl.add(url)  # 已经访问了
                    self.__newUrl.remove(url)  # 删除已经访问过的
                    html = None
                    try:
                        html = requests.get(url)
                    except Exception as e:
                        print(e)
                        continue
                    bs = BeautifulSoup(html.text, features="html.parser")
                    hrefs = bs.find_all("a")
                    for href in hrefs:
                        href = href.get("href")
                        if href != None and href != "" and href[0] == "/":
                            url = self.__rootPath + str(href)
                        else:
                            url = href

                        if href != None and self.__allUrl.__contains__(
                                url) == False:  # 是否已经访问
                            if url.find("blog") >= 0 and url.find(
                                    "sina") >= 0:  # 是否是符合规则的URL
                                self.__newUrl.add(url)  # 符合规则添加到新的URL中
                            else:
                                self.__allUrl.add(url)  # 不符合规则直接扔掉
                            if self.__resultUrl.__contains__(
                                    url
                            ) == False and re.match(
                                    "http://blog\.sina\.com\.cn/s/.*.html\?tj=1",
                                    url) is not None:
                                self.__resultUrl.add(url)
                                self.__dbUtil.insertTbUrl(url)
                                self.__resultL += 1
                                print("目前url条数:" + str(self.__resultL))
        except Exception as e:
            print(e)
        self.getUrl()
        pass
Esempio n. 22
0
 def __init__(self, db):
     '''
        General description:
         This function initializes the database variables and \
         index to refer in functions.
     '''
     DBUtil.__init__(self, db)
     self.collection = db.DeploymentRequest
     self.machineDB = Machine.Machine(db)
     self.versionsDB = Versions.Versions(db)
     self.toolDB = Tool.Tool(db)
     self.passHelper = PasswordHelper.PasswordHelper(key)
     self.statedb =  State.State(db)
Esempio n. 23
0
 def __init__(self, db):
     '''
     General description:
     This function initializes the database variables and \
     index to refer in functions.
     '''
     DBUtil.__init__(self, db)
     self.collection = db.SystemDetails
     self.configDb = Config.Config(mongodb)
     # indexes
     # self.collection.create_index([('version_id', ASCENDING)], unique=False)
     self.collection.create_index([('ip', ASCENDING)], unique=True)
     self.collection.create_index([('hostname', ASCENDING)], unique=True)
Esempio n. 24
0
 def __init__(self, db):
     '''
     General description:
     This function initializes the database variables and \
     index to refer in functions.
     '''
     DBUtil.__init__(self, db)
     self.collection = db.Tool
     self.versionsDB = Versions.Versions(db)
     self.tagDB = Tags.Tags()
     # indexes
     self.collection.create_index([('name', ASCENDING)], unique=True)
     self.collection.create_index([('tag', ASCENDING)], unique=False)
Esempio n. 25
0
 def query_store(self):
     result = []
     # store_id = '9999'
     # LAST_STORE_ID = 'SELECT store_id FROM store WHERE linkman IS NOT NULL ORDER BY store_id DESC LIMIT 1'
     LAST_STORE_ID = "SELECT store_id FROM store WHERE store_homepage='http://shanghaifhm.b2b.hc360.com'"
     # LOAD_STORE = "SELECT store_id,store_homepage FROM store WHERE linkman IS NULL"
     LOAD_STORE = 'SELECT store_id,store_homepage FROM store WHERE linkman IS NULL AND store_id>'
     try:
         store_id = DBUtil().query_one(LAST_STORE_ID)
         result = DBUtil().query_list(LOAD_STORE + store_id)
         # result = DBUtil().query_list(LOAD_STORE)
     except BaseException:
         self.logger.error('查询现有商家出错:' + traceback.format_exc())
     return result
Esempio n. 26
0
 def __init__(self, db):
     '''                                                                    
      General description :                                               
      This function initializes the database variables and \              
      index to refer in functions.                                        
     '''
     DBUtil.__init__(self, db)
     self.collection = db.DeploymentRequestGroup
     self.deploymentRequestDb = DeploymentRequest.DeploymentRequest(db)
     self.machineDB = Machine.Machine(db)
     self.versionsDB = Versions.Versions(db)
     self.toolDB = Tool.Tool(db)
     self.deploymentUnitDB = DeploymentUnit.DeploymentUnit()
     self.deploymentRequestDbCollection = db.DeploymentRequest
     self.statedb = State.State(db)
Esempio n. 27
0
 def __init__(self, db):
     '''
     General description:
     This function initializes the database variables and \
     index to refer in functions.
     '''
     DBUtil.__init__(self, db)
     self.collection = db.UserFavoriteMachine
     self.machineTypeDb = MachineType.MachineType(db)
     self.machineDB = Machine.Machine(db)
     self.userDB = Users.Users(db)
     # indexes
     self.collection.create_index([('user_id', ASCENDING),
                                   ('machine_id', ASCENDING)],
                                  unique=True)
Esempio n. 28
0
 def __init__(self, db):
     '''
         General description:
         This function initializes the database variables and \
         index to refer in functions.
     '''
     DBUtil.__init__(self, db)
     self.collection = db.PermissionGroup
     self.permissiondb = Permissions.Permissions(db)
     self.routesb = Routes.Routes(db)
     # indexes
     self.collection.create_index(
         [('groupname', ASCENDING)],
         unique=True,
     )
Esempio n. 29
0
 def __init__(self, db):
     '''
     General description:
     This function initializes the database variables and \
     index to refer in functions.
     '''
     DBUtil.__init__(self, db)
     self.collection = db.State
     self.deploymentunitapprovalstatusdb = DeploymentUnitApprovalStatus.DeploymentUnitApprovalStatus(
     )
     self.DeploymentFieldsdb = DeploymentFields.DeploymentFields(db)
     self.buildDb = Build.Build()
     # indexes
     self.collection.create_index([('name', ASCENDING),
                                   ('parent_entity_id', ASCENDING)],
                                  unique=True)
Esempio n. 30
0
 def query_products(self):
     result = []
     LOAD_PRODUCT = 'SELECT product_id,product_href FROM product WHERE product_img_0 IS NOT NULL'
     try:
         result = DBUtil().query_list(LOAD_PRODUCT)
     except BaseException:
         self.logger.error('查询所有产品出错:' + traceback.format_exc())
     return result