class xpath_base: hs = url = base_url = db = '' def __init__(self): self.db = DB() def get_field_value(self, value, value_type=None): if value_type == 'img': return self.get_img_url(value) return value def get_all_url(self,website_id): res = self.db.execute('SELECT url FROM le_goods WHERE website_id=%s AND isshow=1',[website_id]) return res.fetchall() def get_img_url(self, url): return urlparse.urljoin(self.base_url, url) def set_defalut(self, spider=None, response=None, text=None): if text!=None: self.hs = Selector(text=text) self.url = 'http://www.ilovedeals.sg' self.base_url = 'http://www.ilovedeals.sg' else: self.hs = Selector(response) self.url = response.url self.base_url = get_base_url(response) if spider == None: item = SgGoodsItem() else: item = eval(spider.xpath_item+'()') for name,value in vars(SgGoodsItem).items(): if name == 'fields': for i in value: if i== 'image_urls' or i == 'images': item[i] = [] else: item[i] = '' return item def run(self, spider=None, response=None, xml=None, text=None): return self.set_defalut(spider=spider, response=response, text=text) def xpath(self): pass def reg(self): pass
def __init__(self): self.db = DB()