def save_product(self, product, category, store_id):
try:
if (not product) or (not category):
return None
result = None
# img = product.get('img')
# 如果产品图片少于8张则填充None
# if len(img) < 8:
# img += [None] * (8 - len(img))
# 通过产品链接查询该产品是否已经存在
QUERY_PRODUCT_ID = "SELECT product_id FROM product WHERE product_href='" + product.get(
'href') + "'"
INSERT_PRODUCT = "INSERT INTO product(product_name,price,quantity_uom_id,currency_uom_id,product_href,product_img_0,product_img_1,product_img_2,product_img_3,product_img_4,product_img_5,product_img_6,product_img_7,description,long_description,category_2nd_id,store_id) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING product_id"
result = DBUtil().query_one(QUERY_PRODUCT_ID)
if result:
return str(result)
result = DBUtil().insert(
INSERT_PRODUCT,
(product.get('title'), product.get('price'),
product.get('unit'), product.get('currency'),
product.get('href'), None, None, None, None, None, None, None,
None, None, None, category, store_id))
except BaseException:
self.logger.error('保存产品信息出错：' + traceback.format_exc())
return str(result)
def __init__(self):
super(MainServer, self).__init__(socket.AF_INET, socket.SOCK_STREAM)
# Getting a connection to the DB.
self.db = DBUtil(DB_CONFIG)
# Creating the necessary tables. for more information see the DDLs above...
self.create_tables()
self.socket_to_address_dict = {}
self.socket_to_username_dict = {}
# Creating dictionary that maps from client to the latest captcha that has been send to him.
self.last_captcha_text = {}
self.captcha_image_generator = ImageCaptcha(fonts=FONTS)
self.captcha_audio_generator = AudioCaptcha(voicedir=VOICES_DIR)
# Listening to the required IP and PORT.
self.bind((BIND_IP, BIND_PORT))
# Limiting the number of clients to BACKLOG.
self.listen(BACKLOG)
# starting a new thread that accepts new clients
Thread(target=self.accept_clients).start()
def __init__(self):
db = mongodb
DBUtil.__init__(self, db)
self.collection = db.Reports
# indexes
self.collection.create_index([('name', ASCENDING)], unique=True)
def producepicture(spm, startdate, enddate):
conn = DBUtil()
tablename = conn.tablename
print("现在处理的spm：", spm)
sql = "SELECT slideend_x,slideend_y,entity_x,entity_y,entity_width,entity_height FROM " \
+tablename+" where spm=%s and dt>=%s and dt<=%s and touch_type=2 order by pos limit 0,100000; "
args = (spm, startdate, enddate)
results = conn.executesearch(sql, args)
conf = config()
processim = np.zeros([conf.heightborder, conf.widthborder],
dtype=int) #高度和宽度
count = 0
for data in results:
if judegzero(data[4], data[5]) != 0:
x = int((data[0] - data[2]) / data[4] *
conf.widthborder) # 鼠标点击位置减去容器框位置除以容器框的宽度
y = int((data[1] - data[3]) / data[5] *
conf.heightborder) #360 120 另外一组是
if judgeoutborder(x, y):
count = count + 1
processim[y, x] = processim[y, x] + 1
if count % 1000 == 0:
print("处理数据进度：", count)
print(str(data))
maxcount = np.max(processim)
print("最大点击次数为：", maxcount)
processim = processim * 255 / maxcount
new_path = "imgs/" + spm + startdate + enddate + ".png"
print("总点击次数为：", count)
if (count >= 1000):
csv_operator.saveexecl(spm, maxcount, count)
#scipy.misc.imsave(new_path, processim)
cv2.imwrite(new_path, processim)
def __init__(self, urls):
self.__rootPath = "https://www.cnblogs.com"
self.__allUrl = set() # 已经访问过的URL
self.__resultUrl = set() # 结果集
self.__newUrl = set(urls) # 新添加的URL
self.__dbUtil = DBUtil()
self.__resultL = 0
pass
def __init__(self, db):
'''
General description:
This function initializes the database variables and \
index to refer in functions.
'''
DBUtil.__init__(self, db)
self.collection = db.Emails
def __init__(self, urls):
self.__rootPath = "http://blog.sina.com.cn"
self.__allUrl = {"http://blog.sina.com.cn/showpic.html"} # 已经访问过的URL
self.__resultUrl = set() # 结果集
self.__newUrl = set(urls) # 新添加的URL
self.__dbUtil = DBUtil()
self.__resultL = 0
pass
def __init__(self, db):
'''
General description:
This function initializes the database variables and
index to refer in functions.
'''
DBUtil.__init__(self, db)
self.collection = db.ContributionGitPushLogs
def __init__(self, db):
'''
General description:
This function initializes the database variables and \
index to refer in functions.
'''
DBUtil.__init__(self, db)
self.collection = db.Sync
self.system_details_db = SystemDetails.SystemDetails(db)
def __init__(self, db):
'''
General description:
This function initializes the database variables and \
index to refer in functions.
'''
DBUtil.__init__(self, db)
self.collection = db.Config
self.passHelper = PasswordHelper.PasswordHelper(key)
def __init__(self, db):
'''
General description:
This function initializes the database variables and \
index to refer in functions.
'''
DBUtil.__init__(self, db)
self.collection = db.Plugins
# indexes
self.collection.create_index([('name', ASCENDING)], unique=True)
def spmlist(startdate, enddate):
conn = DBUtil()
conf = config()
tablename = conf.tablename
sql = "SELECT distinct spm FROM " \
+tablename+" where dt>=%s and dt<=%s and touch_type=2 ; "
args = (startdate, enddate)
results = conn.executesearch(sql, args)
return results
def __init__(self, db):
'''
General description:
This function initializes the database variables and \
index to refer in functions.
'''
DBUtil.__init__(self, db)
self.collection = db.MachineGroups
self.machineDB = Machine.Machine(db)
# indexes
self.collection.create_index([('group_name', ASCENDING)], unique=True)
def __init__(self, db):
'''
General description :
This function initializes the database variables and \
index to refer in functions.
'''
DBUtil.__init__(self, db)
self.collection = db.DeploymentFields
# indexes
self.collection.create_index(
[('parent_entity_id', ASCENDING)], unique=True)
def query_category(self):
category_2nd_id = '9999'
result = []
LAST_CATEGORY_2ND = 'SELECT category_2nd_id FROM product ORDER BY product_id DESC LIMIT 1'
LOAD_CATEGORY = 'SELECT category_2nd_id,category_href FROM category_2nd WHERE category_2nd_id>'
try:
category_2nd_id = DBUtil().query_one(LAST_CATEGORY_2ND)
result = DBUtil().query_list(LOAD_CATEGORY + category_2nd_id)
except BaseException:
self.logger.error('查询分类出错：' + traceback.format_exc())
return result
def Save_To_Database(record):
for onerecord in record:
picturelocation = rootpath + img_path + "/" + onerecord[0]
spm = onerecord[0][:-20]
likelihood = onerecord[1]
conn = DBUtil()
tablename = conf.tablename
sql = "REPLACE INTO " + tablename + " SET spm = %s, Normal_possibility = %s, location = %s;" #注意这行语句只能在spm为主码时使用
args = (spm, str(likelihood), picturelocation)
conn.executesearch(sql, args)
print("save to database over！")
def __init__(self, db):
DBUtil.__init__(self, db)
self.collection = db.Machine
self.passHelper = PasswordHelper.PasswordHelper(key)
self.userdb = Users.Users(db)
self.roleDb = Role.Role(db)
self.tagDB = Tags.Tags()
# indexes
self.collection.create_index([('username', ASCENDING),
('host', ASCENDING)],
unique=True)
def __init__(self):
'''
General description:
This function initializes the database variables and \
index to refer in functions.
'''
db = mongodb
DBUtil.__init__(self, db)
self.collection = db.DeploymentUnitApprovalStatus
# indexes
self.collection.create_index([('name', ASCENDING)], unique=True)
def __init__(self):
'''
General description:
This function initializes the database variables and \
index to refer in functions.
'''
db = mongodb
DBUtil.__init__(self, db)
self.collection = db.Repository
self.passHelper = PasswordHelper.PasswordHelper(key)
# indexes
self.collection.create_index([('name', ASCENDING)], unique=True)
def __init__(self, db):
'''
General description:
This function initializes the database variables and \
index to refer in functions.
'''
DBUtil.__init__(self, db)
self.Collection = db.ToolInstallation
# indexes
self.Collection.create_index([('version_id', ASCENDING),
('machine_id', ASCENDING)],
unique=True)
class BlogRepyilt:
def __init__(self, urls):
self.__rootPath = "http://blog.sina.com.cn"
self.__allUrl = {"http://blog.sina.com.cn/showpic.html"} # 已经访问过的URL
self.__resultUrl = set() # 结果集
self.__newUrl = set(urls) # 新添加的URL
self.__dbUtil = DBUtil()
self.__resultL = 0
pass
def getUrl(self):
urls = self.__newUrl.copy() # 拷贝一份，一边遍历一边删除
try:
for url in urls:
if url.find("showpic") == -1:
self.__allUrl.add(url) # 已经访问了
self.__newUrl.remove(url) # 删除已经访问过的
html = None
try:
html = requests.get(url)
except Exception as e:
print(e)
continue
bs = BeautifulSoup(html.text, features="html.parser")
hrefs = bs.find_all("a")
for href in hrefs:
href = href.get("href")
if href != None and href != "" and href[0] == "/":
url = self.__rootPath + str(href)
else:
url = href
if href != None and self.__allUrl.__contains__(
url) == False: # 是否已经访问
if url.find("blog") >= 0 and url.find(
"sina") >= 0: # 是否是符合规则的URL
self.__newUrl.add(url) # 符合规则添加到新的URL中
else:
self.__allUrl.add(url) # 不符合规则直接扔掉
if self.__resultUrl.__contains__(
url
) == False and re.match(
"http://blog\.sina\.com\.cn/s/.*.html\?tj=1",
url) is not None:
self.__resultUrl.add(url)
self.__dbUtil.insertTbUrl(url)
self.__resultL += 1
print("目前url条数：" + str(self.__resultL))
except Exception as e:
print(e)
self.getUrl()
pass
def __init__(self, db):
'''
General description:
This function initializes the database variables and \
index to refer in functions.
'''
DBUtil.__init__(self, db)
self.collection = db.DeploymentRequest
self.machineDB = Machine.Machine(db)
self.versionsDB = Versions.Versions(db)
self.toolDB = Tool.Tool(db)
self.passHelper = PasswordHelper.PasswordHelper(key)
self.statedb = State.State(db)
def __init__(self, db):
'''
General description:
This function initializes the database variables and \
index to refer in functions.
'''
DBUtil.__init__(self, db)
self.collection = db.SystemDetails
self.configDb = Config.Config(mongodb)
# indexes
# self.collection.create_index([('version_id', ASCENDING)], unique=False)
self.collection.create_index([('ip', ASCENDING)], unique=True)
self.collection.create_index([('hostname', ASCENDING)], unique=True)
def __init__(self, db):
'''
General description:
This function initializes the database variables and \
index to refer in functions.
'''
DBUtil.__init__(self, db)
self.collection = db.Tool
self.versionsDB = Versions.Versions(db)
self.tagDB = Tags.Tags()
# indexes
self.collection.create_index([('name', ASCENDING)], unique=True)
self.collection.create_index([('tag', ASCENDING)], unique=False)
def query_store(self):
result = []
# store_id = '9999'
# LAST_STORE_ID = 'SELECT store_id FROM store WHERE linkman IS NOT NULL ORDER BY store_id DESC LIMIT 1'
LAST_STORE_ID = "SELECT store_id FROM store WHERE store_homepage='http://shanghaifhm.b2b.hc360.com'"
# LOAD_STORE = "SELECT store_id,store_homepage FROM store WHERE linkman IS NULL"
LOAD_STORE = 'SELECT store_id,store_homepage FROM store WHERE linkman IS NULL AND store_id>'
try:
store_id = DBUtil().query_one(LAST_STORE_ID)
result = DBUtil().query_list(LOAD_STORE + store_id)
# result = DBUtil().query_list(LOAD_STORE)
except BaseException:
self.logger.error('查询现有商家出错：' + traceback.format_exc())
return result
def __init__(self, db):
'''
General description :
This function initializes the database variables and \
index to refer in functions.
'''
DBUtil.__init__(self, db)
self.collection = db.DeploymentRequestGroup
self.deploymentRequestDb = DeploymentRequest.DeploymentRequest(db)
self.machineDB = Machine.Machine(db)
self.versionsDB = Versions.Versions(db)
self.toolDB = Tool.Tool(db)
self.deploymentUnitDB = DeploymentUnit.DeploymentUnit()
self.deploymentRequestDbCollection = db.DeploymentRequest
self.statedb = State.State(db)
def __init__(self, db):
'''
General description:
This function initializes the database variables and \
index to refer in functions.
'''
DBUtil.__init__(self, db)
self.collection = db.UserFavoriteMachine
self.machineTypeDb = MachineType.MachineType(db)
self.machineDB = Machine.Machine(db)
self.userDB = Users.Users(db)
# indexes
self.collection.create_index([('user_id', ASCENDING),
('machine_id', ASCENDING)],
unique=True)
def __init__(self, db):
'''
General description:
This function initializes the database variables and \
index to refer in functions.
'''
DBUtil.__init__(self, db)
self.collection = db.PermissionGroup
self.permissiondb = Permissions.Permissions(db)
self.routesb = Routes.Routes(db)
# indexes
self.collection.create_index(
[('groupname', ASCENDING)],
unique=True,
)
def __init__(self, db):
'''
General description:
This function initializes the database variables and \
index to refer in functions.
'''
DBUtil.__init__(self, db)
self.collection = db.State
self.deploymentunitapprovalstatusdb = DeploymentUnitApprovalStatus.DeploymentUnitApprovalStatus(
)
self.DeploymentFieldsdb = DeploymentFields.DeploymentFields(db)
self.buildDb = Build.Build()
# indexes
self.collection.create_index([('name', ASCENDING),
('parent_entity_id', ASCENDING)],
unique=True)
def query_products(self):
result = []
LOAD_PRODUCT = 'SELECT product_id,product_href FROM product WHERE product_img_0 IS NOT NULL'
try:
result = DBUtil().query_list(LOAD_PRODUCT)
except BaseException:
self.logger.error('查询所有产品出错：' + traceback.format_exc())
return result
