def scrape(self, downloadQueue):
try:
self.region = downloadQueue[2]
self.type = downloadQueue[3]
self.value = downloadQueue[4].encode('utf-8')
region = Model_Static_Region()
reg = region.getText(self.region)
type = self.type
asin = self.value
if (type == Model_Static_DownloadQueue_Type.PRODUCT_OFFER):
self.offerScraper = Model_Scraper_Product_Offer(reg)
data = self.offerScraper.scrape(asin)
# 操作scrape表，写入抓取日志
if (self.offerScraper.hasScrapes()):
# 执行写入操作
try:
self.saveScrapes(self.offerScraper.getScrapes(), downloadQueue)
except:
pass
scrapeSuccess = False
if(data):
scrapeSuccess = True
for index in range(len(data)):
for items in data[index]:
# print (items)
if (items['seller_id'] != ''):
# 插入amazon_seller_product_offer表
try:
if (items['item_id']):
self.getAmazonSellerProductOfferMapper().save(reg, asin, items)
except Exception as err:
print (err)
# 插入amazon_seller表
try:
self.getAmazonSellerMapper().save(reg, items)
except Exception as err:
print (err)
# 插入amazon_seller_product表
try:
self.getAmazonSellerProductMapper().save(reg, asin, items)
except Exception as err:
print (err)
else:
continue
if (scrapeSuccess):
return Model_Static_DownloadQueue_Status.SCRAPED
else:
return Model_Static_DownloadQueue_Status.FAILED
elif (data == False):
return Model_Static_DownloadQueue_Status.SCRAPED_WRONG
else:
return Model_Static_DownloadQueue_Status.SCRAPED_NO_DATA
except Exception as err:
print (err)
return Model_Static_DownloadQueue_Status.FAILED
def scrape(self, downloadQueue):
try:
self.region = downloadQueue[2]
self.type = downloadQueue[3]
self.value = downloadQueue[4].encode('utf-8')
region = Model_Static_Region()
reg = region.getText(self.region)
asin = self.value
self.scraper = Model_Scraper_Product_Base(reg)
data = self.scraper.scrape(asin)
# 操作scrape表，写入抓取日志
if (self.scraper.hasScrapes()):
try:
# 执行写入操作
self.saveScrapes(self.scraper.getScrapes(), downloadQueue)
except:
pass
position = 0
scrapeSuccess = False
if(data):
# print (data)
scrapeSuccess = True
# 表amazon_product
result = self.getAmazonProductMapper().save(reg, asin, data)
# 表amazon_product_images
if (result):
for i in data['images']:
try:
self.getAmazonProductImageMapper().save(reg, asin, i, position)
position += 1
except Exception as err:
print (err)
if (scrapeSuccess):
return Model_Static_DownloadQueue_Status.SCRAPED
else:
return Model_Static_DownloadQueue_Status.FAILED
elif (data == False):
return Model_Static_DownloadQueue_Status.SCRAPED_WRONG
else:
# 若未抓出数据，进行相应操作
return Model_Static_DownloadQueue_Status.SCRAPED_NO_DATA
except Exception as err:
print (err)
return Model_Static_DownloadQueue_Status.FAILED
def mobile_scrape(self, mobiledownloadQueue):
try:
region = Model_Static_Region()
self.region = mobiledownloadQueue[2]
reg = region.getText(self.region)
keywords = mobiledownloadQueue[4].encode('utf-8')
self.scraper = Model_Scraper_Mobile_Product_Base(reg)
data = self.scraper.scrape(reg, keywords)
scrapeSuccess = False
if (data[-1]["total"] > 0):
scrapeSuccess = True
# 存储产品基本信息
for index in range(len(data) - 1):
for page_data in data[index]:
try:
result = self.amazonProductMapper.saveFromKeywordsProduct(reg, page_data)
if (result):
# 插入或更新图片信息0-
try:
asin = page_data['asin']
if (page_data['image']):
self.amazonProductImageMapper.save_img(reg, asin, page_data)
self.uploadMobileQueueMapper.save(self.region, 0, asin)
except Exception as err:
print (err)
except Exception as err:
print (err)
# 存储关键词基本信息
try:
self.mobilekeywordsMapper.save(reg, keywords, str(data[-1]["total"]))
except:
print ("Mobile_Keywords insert error")
if (scrapeSuccess):
return Model_Static_DownloadQueue_Status.SCRAPED
# 更改队列状态
else:
return Model_Static_DownloadQueue_Status.FAILED
else:
return Model_Static_DownloadQueue_Status.FAILED
except Exception as err:
print (err)
def scrape(self, downloadQueue):
try:
self.region = downloadQueue[2]
region = Model_Static_Region()
reg = region.getText(self.region)
merchant_id = downloadQueue[4]
self.scraper = Model_Scraper_Seller_Base(reg)
data = self.scraper.scrape(merchant_id)
# # 数据库连接
# scrape = Model_Mapper_Scrape()
# amazon = scrape.amazon
# mapper = Model_Mapper_Scrape().mapper
if (data):
print(data)
# 先查找数据库ASIN，若有则执行更新操作，若无则执行插入操作
result = self.getAmazonSellerMapper().save_seller(
reg, merchant_id, data)
# sql = amazon.seller_select_sql_joint(reg, merchant_id)
# result = mapper.select(sql)
# if (result):
# sql = amazon.sellerinfo_update_sql_joint(reg, merchant_id, data)
# result = mapper.update(sql)
# else:
# sql = amazon.sellerinfo_insert_sql_joint(reg, merchant_id, data)
# result = mapper.update(sql)
if (result):
return Model_Static_DownloadQueue_Status.SCRAPED
else:
return Model_Static_DownloadQueue_Status.FAILED
else:
# 若未抓出数据，进行相应操作
return Model_Static_DownloadQueue_Status.FAILED
except Exception as err:
print(err)
def scrapeTopReviewer(self, downloadQueue):
try:
value = downloadQueue[4]
try:
value = value.split(":")
except Exception as err:
print(err)
if (len(value) == 2):
begin = int(value[0])
end = int(value[1])
else:
begin = 1
end = int(value[0])
region = Model_Static_Region().getText(downloadQueue[2])
self.scraper = Model_Scraper_TopReviewer(region)
if (begin < 1):
begin = 1
if (end > 1000):
end = 1000
data = self.scraper.scrape(begin, end + 1)
# 数据库初始化操作，判断数据有效性，然后对数据进行后续操作
amazon = Model_Mapper_Amazon()
# 连接数据库
db = Model_Mapper_Connect('localhost', 3306, 'root', '123123',
'scraper', 'utf8')
conn = db.connects()
mapper = Model_Mapper_Mysql(conn)
if (data):
rankBegin = begin * 10 - 9
rankEnd = end * 10
sql = amazon.TopReviewer_delete_sql_joint(
region, rankBegin, rankEnd)
mapper.delete(sql)
for items in data:
for item in items:
# print (item)
sql = amazon.TopReviewer_insert_sql_joint(region, item)
mapper.insert(sql)
return Model_Static_DownloadQueue_Status.SCRAPED
elif (data == ""):
return Model_Static_DownloadQueue_Status.SCRAPED_NO_DATA
else:
return Model_Static_DownloadQueue_Status.FAILED
except Exception as err:
print(err)
def scrapeFirst(self, downloadQueue):
try:
self.region = downloadQueue[2]
self.type = downloadQueue[3]
self.value = downloadQueue[4].encode('utf-8')
region = Model_Static_Region()
reg = region.getText(self.region)
keywords = self.value
self.scraper = Model_Scraper_Keywords_First(reg)
data = self.scraper.scraper(keywords)
# 操作scrape表，写入抓取日志
if (self.scraper.hasScrapes()):
# 执行写入操作
self.saveScrapes(self.scraper.getScrapes(), downloadQueue)
node_id = 0
scrapeSuccess = False
if (data):
print(len(data))
# print (data)
try:
# 在插入数据库操作前，先将该广告关键词表中的数据清除
self.getAmazonKeywordsAdMapper().clearAdData(
reg, keywords, str(node_id))
except Exception as err:
print(err)
try:
# 在插入数据库操作前，先将该普通排名关键词下的数据清除
self.getAmazonKeywordsRankMapper().clearRankData(
reg, keywords, str(node_id))
except Exception as err:
print(err)
# 更新新的数据
# print (data)
if (data[-1]["total"] > 0):
scrapeSuccess = True
rank = 1 # 绝对排名
position = 1 # 广告绝对位置
try:
for index in range(len(data) - 1):
# 处理广告
if (data[index]['sponsor'] == 1):
if (data[index]['sponsor_position_type'] ==
'right'):
ad_position_type = Model_Static_Amazon_Product_Keywords_Ad_AdPositionType(
).RIGHT
elif (data[index]['sponsor_position_type'] ==
'top'):
ad_position_type = Model_Static_Amazon_Product_Keywords_Ad_AdPositionType(
).TOP
else:
ad_position_type = Model_Static_Amazon_Product_Keywords_Ad_AdPositionType(
).BOTTOM
ad_position = data[index]['sponsor_position']
try:
result = self.getAmazonKeywordsAdMapper(
).save(reg, keywords, node_id, ad_position,
ad_position_type, position,
data[index])
except Exception as err:
print(err)
if (result):
position += 1
# 处理自然排名
else:
try:
result = self.getAmazonKeywordsRankMapper(
).save(reg, keywords, node_id, rank,
data[index])
except Exception as err:
print(err)
if (result):
rank += 1
except Exception as err:
print(err)
# 若未抓出数据，进行相应操作
elif (data[-1]["total"] == 0):
scrapeSuccess = True
# 存储产品基本信息
for index in range(len(data) - 1):
try:
self.saveAmazonProduct(reg, data[index])
except:
print("saveAmazonProduct error")
# 存储关键词基本信息
if (data[-1]["total"] > 0):
try:
self.getKeywordsMapper().save(reg, keywords,
str(data[-1]["total"]))
except Exception as err:
print(err)
if (scrapeSuccess):
return Model_Static_DownloadQueue_Status.SCRAPED
else:
return Model_Static_DownloadQueue_Status.FAILED
elif (data == False):
return Model_Static_DownloadQueue_Status.SCRAPED_WRONG
else:
# 若未抓出数据，进行相应操作
return Model_Static_DownloadQueue_Status.SCRAPED_NO_DATA
except Exception as err:
print(err)
def scrape(self, downloadQueue):
self.region = downloadQueue[2]
self.type = downloadQueue[3]
self.value = downloadQueue[4].encode('utf-8')
region = Model_Static_Region()
reg = region.getText(self.region)
keywords = self.value
self.scraper = Model_Scraper_Keywords(reg)
results = self.scraper.scraper(keywords)
# 操作scrape表，写入抓取日志
if (self.scraper.hasScrapes()):
# 执行写入操作
try:
self.saveScrapes(self.scraper.getScrapes(), downloadQueue)
except:
pass
node_id = 0
scrapeSuccess = False
if (results):
# # 数据库初始化操作，判断数据有效性，然后对数据进行后续操作
# amazon = Model_Mapper_Amazon()
# # 连接数据库
# db = Model_Mapper_Connect('localhost', 3306, 'root', '123123', 'scraper', 'utf8')
# conn = db.connects()
# mapper = Model_Mapper_Mysql(conn)
try:
# 在插入数据库操作前，先将该广告关键词表中的数据清除
# sql = amazon.keywords_ad_delete_sql_joint(reg, keywords, str(node_id))
# mapper.delete(sql)
self.getAmazonKeywordsAdMapper().clearAdData(
reg, keywords, str(node_id))
except Exception as err:
print(err)
try:
# 在插入数据库操作前，先将该普通排名关键词下的数据清除
# sql = amazon.keywords_rank_delete_sql_joint(reg, keywords, str(node_id))
# mapper.delete(sql)
self.getAmazonKeywordsRankMapper().clearRankData(
reg, keywords, str(node_id))
except Exception as err:
print(err)
# 更新新的数据
# print (results)
for data in results:
# print (data[-1])
if (data[-1]["total"] > 0):
scrapeSuccess = True
rank = 1 # 绝对排名
position = 1 # 广告绝对位置
try:
for index in range(len(data) - 1):
# 处理广告
if (data[index]['sponsor'] == 1):
if (data[index]['sponsor_position_type'] ==
'right'):
ad_position_type = Model_Static_Amazon_Product_Keywords_Ad_AdPositionType(
).RIGHT
elif (data[index]['sponsor_position_type'] ==
'top'):
ad_position_type = Model_Static_Amazon_Product_Keywords_Ad_AdPositionType(
).TOP
else:
ad_position_type = Model_Static_Amazon_Product_Keywords_Ad_AdPositionType(
).BOTTOM
ad_position = data[index]['sponsor_position']
try:
result = self.getAmazonKeywordsAdMapper(
).save(reg, keywords, node_id, ad_position,
ad_position_type, position,
data[index])
except Exception as err:
print(err)
# sql = amazon.keywords_ad_insert_sql_joint(reg, keywords, node_id, ad_position, ad_position_type, position, data[index])
# result = mapper.insert(sql)
if (result):
position += 1
# 处理自然排名
else:
try:
result = self.getAmazonKeywordsRankMapper(
).save(reg, keywords, node_id, rank,
data[index])
except Exception as err:
print(err)
# sql = amazon.keywords_rank_insert_sql_joint(reg, keywords, node_id, rank, data[index])
# result = mapper.insert(sql)
if (result):
rank += 1
except Exception as err:
print(err)
# 若未抓出数据，进行相应操作
elif (data[-1]["total"] == 0):
scrapeSuccess = True
# 存储产品基本信息
for index in range(len(data) - 1):
# print (data[index])
try:
self.saveAmazonProduct(reg, data[index])
except:
print("saveAmazonProduct error")
# for index in range(len(data) - 1):
# sql = amazon.keywords_product_insert_sql_joint(reg, data[index])
################################xiugaiasdsadasdfsadfas###############################
# 存储关键词基本信息
if (results[0][-1]["total"] > 0):
try:
self.getKeywordsMapper().save(reg, keywords,
str(results[0][-1]["total"]))
except:
print("Keywords insert error")
# sql = amazon.keywordsObject_select_sql_joint(reg, keywords)
# result = mapper.select(sql)
# if (result):
# sql = amazon.keywordsObject_update_sql_joint(reg, keywords, str(results[0][-1]))
# result = mapper.update(sql)
# else:
# sql = amazon.keywordsObject_insert_sql_joint(reg, keywords, str(results[0][-1]))
# result = mapper.insert(sql)
# if (result):
# scrapeSuccess = True
# conn.close()
if (scrapeSuccess == True):
return Model_Static_DownloadQueue_Status.SCRAPED
else:
return Model_Static_DownloadQueue_Status.FAILED
elif (results == False):
return Model_Static_DownloadQueue_Status.SCRAPED_WRONG
else:
return Model_Static_DownloadQueue_Status.FAILED
def processMobileUploadQueues(self, mobileuploadQueues, region):
data = {'region': region}
for mobileuploadQueue in mobileuploadQueues:
queueRegion = Model_Static_Region().getText(mobileuploadQueue[1])
queueValue = mobileuploadQueue[3]
result = False
if (str(mobileuploadQueue[2]).isdigit()):
type = mobileuploadQueue[2]
# 产品页 0
if (type == Model_Static_DownloadQueue_Type.PRODUCT):
result = self.getProductService(
).getAmazonProductUploadData(queueRegion, queueValue, None)
# # 前5页关键词 3
# elif (type == Model_Static_DownloadQueue_Type.KEYWORDS):
# value = queueValue
# keywords = value
# result = self.getKeywordsService().getAmazonKeywordsRankUploadData(queueRegion, keywords)
# # 首页关键词 4
# elif (type == Model_Static_DownloadQueue_Type.KEYWORDS_INFO):
# value = queueValue
# keywords = value
# result = self.getKeywordsService().getAmazonKeywordsInfoUploadData(queueRegion, keywords)
# # offer页面计算库存 1
# elif (type == Model_Static_DownloadQueue_Type.PRODUCT_OFFER):
# result = self.getProductService().getAmazonProductUploadData(queueRegion, queueValue, 'OFFER')
# # elif (type == Model_Static_DownloadQueue_Type.PRODUCT_REVIEW):
# # pass
# elif (type == Model_Static_DownloadQueue_Type.SELLER):
# merchant_id = queueValue
# result = self.getSellerService().getAmazonSellerUploadData(queueRegion, merchant_id)
# elif (type == Model_Static_DownloadQueue_Type.SELLER_PRODUCT):
# merchant_id = queueValue
# result = self.getSellerService().getAmazonSellerUploadData(queueRegion, merchant_id, True)
# elif (type == Model_Static_DownloadQueue_Type.TOP_REVIEWER):
# value = queueValue
# value = value.split(":")
# if (len(value) == 2):
# begin = value[0]
# end = value[1]
# else:
# begin = 1
# end = value[0]
# result = self.getCustomerService().getAmazonTopReviewerUploadData(queueRegion, begin, end)
else:
pass
# 整合数据
data['region_data'] = {
mobileuploadQueue[0]: {
"data": result,
'region': mobileuploadQueue[1],
'type': mobileuploadQueue[2],
'value': mobileuploadQueue[3],
}
}
# print data
uploadService = Service_Upload()
result = uploadService.upload(data, 'UploadQueue')
# print (result)
if (result):
# 开始更改下载队列中的上传状态和上传次数
amazon = Model_Mapper_Amazon()
# 连接数据库
db = Model_Mapper_Connect('localhost', 3306, 'root', '123123',
'scraper', 'utf8')
conn = db.connects()
mapper = Model_Mapper_Mysql(conn)
# 根据云服务器返回数据进行判断，这里还要改进
for upload_queue_id, subresult in result.items():
# subresult = 1
if (subresult == True):
sql = amazon.MobileUploadQueue_select_sql_joint(
str(upload_queue_id))
uploadQueue = mapper.select(sql)
if (uploadQueue):
sql = amazon.MobileUploadQueue_delete_sql_joint(
str(upload_queue_id))
mapper.delete(sql)
conn.close()
def uploadDownloadQueues(self, downloadQueues, region):
data = {'region': region}
# 开始更改下载队列中的上传状态和上传次数
amazon = Model_Mapper_Amazon()
# 连接数据库
db = Model_Mapper_Connect('localhost', 3306, 'root', '123123',
'scraper', 'utf8')
conn = db.connects()
mapper = Model_Mapper_Mysql(conn)
for downloadQueue in downloadQueues:
queueRegion = Model_Static_Region().getText(downloadQueue[2])
queueValue = downloadQueue[4]
result = False
if (downloadQueue[5] == Model_Static_DownloadQueue_Status.SCRAPED):
type = downloadQueue[3]
# 产品页 0
if (type == Model_Static_DownloadQueue_Type.PRODUCT or type
== Model_Static_DownloadQueue_Type.PRODUCT_INFO):
result = self.getProductService(
).getAmazonProductUploadData(queueRegion, queueValue, None)
# # 前5页关键词 3 或首页关键词
elif (type == Model_Static_DownloadQueue_Type.KEYWORDS or type
== Model_Static_DownloadQueue_Type.KEYWORDS_INFO):
value = queueValue
keywords = value
result = self.getKeywordsService(
).getAmazonKeywordsUploadData(queueRegion, keywords)
# offer页面计算库存 1
elif (type == Model_Static_DownloadQueue_Type.PRODUCT_OFFER):
result = self.getProductService(
).getAmazonProductUploadData(queueRegion, queueValue,
"OFFER")
elif (type == Model_Static_DownloadQueue_Type.PRODUCT_REVIEW):
pass
elif (type == Model_Static_DownloadQueue_Type.SELLER):
merchant_id = queueValue
result = self.getSellerService().getAmazonSellerUploadData(
queueRegion, merchant_id)
elif (type == Model_Static_DownloadQueue_Type.SELLER_PRODUCT):
merchant_id = queueValue
result = self.getSellerService().getAmazonSellerUploadData(
queueRegion, merchant_id, True)
elif (type == Model_Static_DownloadQueue_Type.TOP_REVIEWER):
value = queueValue
value = value.split(":")
if (len(value) == 2):
begin = value[0]
end = value[1]
else:
begin = 1
end = value[0]
result = self.getCustomerService(
).getAmazonTopReviewerUploadData(queueRegion, begin, end)
else:
pass
# 整合数据
data['region_data'] = {
downloadQueue[0]: {
"data": result,
'region': str(downloadQueue[2]),
'type': str(downloadQueue[3]),
'value': downloadQueue[4],
"ac_download_queue_id": str(downloadQueue[1]),
"status": str(downloadQueue[5])
}
}
# print data
uploadService = Service_Upload()
result = uploadService.upload(data)
# print (result)
if (result):
# 根据云服务器返回数据进行判断，这里还要改进
for download_queue_id, subresult in result.items():
# download_queue_id = downloadQueue[0]
# subresult = 1
sql = amazon.DownloadQueue_select_sql_joint(
download_queue_id)
downloadQueue = mapper.select(sql)
if (len(downloadQueue) > 0):
id = downloadQueue[0][0]
count = downloadQueue[0][9]
count += 1
status = Model_Static_DownloadQueue_UploadStatus(
).PENDING
if (subresult):
status = Model_Static_DownloadQueue_UploadStatus(
).UPLOADED
else:
if (count > 2):
status = Model_Static_DownloadQueue_UploadStatus(
).FAILED
sql = amazon.DownloadQueue_update_sql_joint(
status, count, id)
mapper.update(sql)
conn.close()
def scrapeProduct(self, downloadQueue):
try:
self.region = downloadQueue[2]
region = Model_Static_Region()
reg = region.getText(self.region)
merchant_id = downloadQueue[4]
self.scraper = Model_Scraper_Seller_Product(reg)
results = self.scraper.scrape(merchant_id)
if (results):
# print (results)
# 数据库初始化操作，判断数据有效性，然后对数据进行后续操作
amazon = Model_Mapper_Amazon()
# 连接数据库
db = Model_Mapper_Connect('localhost', 3306, 'root', '123123',
'scraper', 'utf8')
conn = db.connects()
mapper = Model_Mapper_Mysql(conn)
# 插入卖家产品信息前先将卖家信息插入卖家信息表
self.getAmazonSellerMapper().save_productsseller(
reg, merchant_id)
# sql = amazon.seller_select_sql_joint(reg, merchant_id)
# seller = mapper.select(sql)
# if (seller):
# pass
# else:
# sql = amazon.product_seller_insert_sql_joint(reg, merchant_id)
# mapper.insert(sql)
# 插入卖家产品信息
rank = 1
# 更新排名前现将该店铺产品rank清空
self.getAmazonSellerMapper().save_updaterank(reg, merchant_id)
# sql = amazon.sellerproducts_rankupdate_sql_joint(reg, merchant_id)
# mapper.update(sql)
for items in results:
# print (result)
for item in items:
asin = item['asin']
sql = amazon.sellerproduct_select_sql_joint(
reg, asin, merchant_id)
result = mapper.select(sql)
if (result):
sql = amazon.sellerproducts_update_sql_joint(
reg, asin, merchant_id, rank, item)
result = mapper.update(sql)
else:
sql = amazon.sellerproducts_insert_sql_joint(
reg, merchant_id, rank, item)
result = mapper.insert(sql)
if (result):
rank += 1
# 插入产品数据(amazon_product表)
sql = amazon.product_select_sql_joint(reg, asin)
result = mapper.select(sql)
if (result):
sql = amazon.products_update_sql_joint(
reg, asin, item)
result = mapper.update(sql)
else:
sql = amazon.products_insert_sql_joint(
reg, asin, item)
result = mapper.insert(sql)
if (result):
# 插入图片数据(amazon_product_image表)
if (item['image']):
sql = amazon.product_image_select_sql_joint(
reg, asin)
result = mapper.select(sql)
if (result):
pass
else:
sql = amazon.product_image_insert_sql_joint(
reg, asin, item)
mapper.insert(sql)
conn.close()
if (result):
return Model_Static_DownloadQueue_Status.SCRAPED
else:
return Model_Static_DownloadQueue_Status.FAILED
else:
return Model_Static_DownloadQueue_Status.FAILED
except Exception as err:
print(err)