class xpath_base:
hs = url = base_url = db = ''
def __init__(self):
self.db = DB()
def get_field_value(self, value, value_type=None):
if value_type == 'img':
return self.get_img_url(value)
return value
def get_all_url(self,website_id):
res = self.db.execute('SELECT url FROM le_goods WHERE website_id=%s AND isshow=1',[website_id])
return res.fetchall()
def get_img_url(self, url):
return urlparse.urljoin(self.base_url, url)
def set_defalut(self, spider=None, response=None, text=None):
if text!=None:
self.hs = Selector(text=text)
self.url = 'http://www.ilovedeals.sg'
self.base_url = 'http://www.ilovedeals.sg'
else:
self.hs = Selector(response)
self.url = response.url
self.base_url = get_base_url(response)
if spider == None:
item = SgGoodsItem()
else:
item = eval(spider.xpath_item+'()')
for name,value in vars(SgGoodsItem).items():
if name == 'fields':
for i in value:
if i== 'image_urls' or i == 'images':
item[i] = []
else:
item[i] = ''
return item
def run(self, spider=None, response=None, xml=None, text=None):
return self.set_defalut(spider=spider, response=response, text=text)
def xpath(self):
pass
def reg(self):
pass
def __init__(self):
self.db = DB()
