def readAliexpress(self, request, old_obj, obj):
#判断新老url是否相等
new_SourceURL = ''
if obj is not None and obj.SourceURL is not None and obj.SourceURL.strip(
) != '':
new_SourceURL = self.format_urls(obj.SourceURL)
else:
return
old_SourceURL = ''
if old_obj is not None and old_obj.SourceURL is not None and old_obj.SourceURL.strip(
) != '':
old_SourceURL = self.format_urls(old_obj.SourceURL)
if old_SourceURL.strip() == new_SourceURL.strip(
) and old_obj.SourcePicPath is not None and old_obj.SourcePicPath.strip(
) != '':
return
aliexpressurls = new_SourceURL
logger = logging.getLogger(
'sourceDns.webdns.views') #刚才在setting.py中配置的logger
logger.error(
"workfunc_Wishworkfunc_Wishworkfunc_Wish BEGIN obj = %s aliexpressurls =%s"
% (obj, aliexpressurls))
data_bytes = None
opener = proxy.get_proxy()
try:
req = urllib2.Request(aliexpressurls)
data_bytes = opener.open(req, timeout=30).read() #.decode('gbk')
except urllib2.HTTPError, e:
messages.error(request, '反向链接读取错误.%s' % e.reason)
return
def getPriceSection(id):
url = "https://www.aliexpress.com/item/abc/%s.html" % (id)
req = urllib2.Request(url)
res = proxy.get_proxy().open(req, timeout=60).read()
html = etree.HTML(res)
discountprice = html.xpath('//*[@id="j-sku-discount-price"]/span[1]')
pricecode = html.xpath(
'//*[@id="j-product-detail-bd"]/div[1]/div/div[2]/div[2]/div/div[2]/div/div[1]/span[1]'
)
if len(discountprice) == 1:
lowprice = html.xpath('//*[@id="j-sku-discount-price"]/span[1]')
hightprice = html.xpath('//*[@id="j-sku-discount-price"]/span[2]')
showprice = ('[%s,%s]' %
(lowprice[0].text, hightprice[0].text)).replace(
' ', '')
else:
price = html.xpath('//*[@id="j-sku-price"]/span[1]')
if len(price) == 1:
lowprice = html.xpath('//*[@id="j-sku-price"]/span[1]')
hightprice = html.xpath('//*[@id="j-sku-price"]/span[2]')
showprice = ('[%s,%s]' %
(lowprice[0].text, hightprice[0].text)).replace(
' ', '')
else:
price = html.xpath('//*[@id="j-sku-price"]')
showprice = '[%s]' % (price[0].text)
return showprice
def read1688_2(self, request, old_obj, obj):
#判断新老url是否相等
new_SourceURL = ''
if obj is not None and obj.SupplierPUrl1 is not None and obj.SupplierPUrl1.strip(
) != '':
new_SourceURL = self.format_urls(obj.SupplierPUrl1)
else:
return
old_SourceURL = ''
if old_obj is not None and old_obj.SupplierPUrl1 is not None and old_obj.SupplierPUrl1.strip(
) != '':
old_SourceURL = self.format_urls(old_obj.SupplierPUrl1)
if old_SourceURL.strip() == new_SourceURL.strip(
) and old_obj.SourcePicPath2 is not None and old_obj.SourcePicPath2.strip(
) != '':
#messages.error(request,'重复供应商URL= %s '%(new_SourceURL.strip()))
return
data_bytes = None
opener = proxy.get_proxy()
#取网站数据
try:
# req = urllib2.Request(www1688urls)
# data_bytes = opener.open(req, timeout = 30).read().decode('gbk')
urllib2.install_opener(urlopener)
req = urllib2.Request(new_SourceURL)
data_bytes = urllib2.urlopen(req, timeout=30).read().decode('gbk')
except urllib2.HTTPError, e:
messages.error(request, '供货商商品链接二读取错误1.%s' % e.reason)
return
def read1688(self, request, obj, url):
#判断新老url是否相等
#obj_original = self.model.objects.get(pk=obj.pk)
#if obj_original.SourceURL == obj.SourceURL :
#return
self.logger.error("readAmazonreadAmazonreadAmazon")
if url is None or url.strip() == '':
return
#self.logger.error("SourceURL0SourceURL0SourceURL0 =%s %s "%(obj.SourceURL0,obj.SourceURL))
#去掉？后面的
urlNew = url
if url is not None and url.find('?') >= 0:
urlNew = url.split(r'?')[0]
www1688urls = urlNew
data_bytes = None
opener = proxy.get_proxy()
#取网站数据
try:
#data_bytes = urllib2.urlopen(www1688urls, timeout = 10).read().decode('gbk')
req = urllib2.Request(www1688urls)
data_bytes = opener.open(req, timeout=30).read().decode('gbk')
except urllib2.HTTPError, e:
messages.error(request, '供货商商品链接一读取错误.%s' % e.reason)
return
def getTheResultnum(id):
req = urllib2.Request(compirePrice.haveNextPage(id, 1))
res = proxy.get_proxy().open(req, timeout=60).read()
jdata = json.loads(res)
total = jdata['page']['total']
current = jdata['page']['current']
print 'total:%s 页,current:%s 页' % (int(total), int(current))
num = 0
for x in range(int(total)):
req = urllib2.Request(compirePrice.haveNextPage(id, x + 1))
res = proxy.get_proxy().open(req, timeout=60).read()
jdata = json.loads(res)
records = jdata['records']
for record in records:
thisdate = (pptime.datetime.strptime(
record['date'],
"%d %b %Y %H:%M")).strftime('%Y-%m-%d %H:%M')
if thisdate >= (pptime.datetime.utcnow() + pptime.timedelta(
days=-15)).strftime('%Y-%m-%d %H:%M'):
num += 1
return num
def read_ebay_detail(self, soup, obj):
details_bytes = None
detailsurl = ''
for atemp in soup.find_all('a'):
detailsurl = atemp.get('href', '')
urlstring = ('%s' % atemp.string).split(' ')
if urlstring[-1] == 'sold' and detailsurl.find('http://') == 0:
break
if detailsurl:
try:
detreq = urllib2.Request(detailsurl)
details_bytes = proxy.get_proxy().open(detreq,
timeout=60).read()
except urllib2.HTTPError, e:
messages.error(request,
'反向链接数据获取错误，请选择手动上传图片或录入数据.%s' % e.reason)
return
except urllib2.URLError, e:
messages.error(request,
'反向链接数据获取错误，请选择手动上传图片或录入数据.%s' % e.reason)
return
def readWish(self, request, old_obj, obj):
#判断新老url是否相等
new_SourceURL = ''
if obj is not None and obj.SourceURL is not None and obj.SourceURL.strip(
) != '':
new_SourceURL = self.format_urls(obj.SourceURL)
else:
return
old_SourceURL = ''
if old_obj is not None and old_obj.SourceURL is not None and old_obj.SourceURL.strip(
) != '':
old_SourceURL = self.format_urls(old_obj.SourceURL)
if old_SourceURL.strip() == new_SourceURL.strip(
) and old_obj.SourcePicPath is not None and old_obj.SourcePicPath.strip(
) != '':
#messages.error(request,'重复URL= %s '%(new_SourceURL.strip()))
return
wishurls = new_SourceURL
wishpicurls = None
data_bytes = None
jo = None
#取网站数据
opener = proxy.get_proxy()
try:
req = urllib2.Request(wishurls)
data_bytes = opener.open(req, timeout=30).read()
#logger.error("workfunc_Wishworkfunc_Wishworkfunc_Wish 1111 obj = %s wishurls =%s"%(obj,wishurls))
#self.message_user(request,data_bytes)
except urllib2.HTTPError, e:
messages.error(request, '反向链接读取错误.%s' % e.reason)
return