def __init__(self, *a, **kw): super(ChomeSpider, self).__init__(*a, **kw) dispatcher.connect(self.spider_closed, signals.spider_closed) terminal = DatabaseTerminal(sys.argv, self.name) self.d = terminal.get_arguments() self.xml = CommonXml() self.exc = ZmagsException(5) if self.d['database']: self.database = Database() self.database.connect() self.products, self.no_urls = self.database.select_products(self.d['catalog_id'], self.d['product_id']) self.database.disconnect() else: self.get_lists_from_excel() self.add_properties(self.xml) self.images_store = "/" + settings['IMAGES_STORE'] self.total = len(self.no_urls['product_ids'])
def __init__(self, *a, **kw): super(GuitarCenterSpider, self).__init__(*a, **kw) dispatcher.connect(self.spider_closed, signals.spider_closed) terminal = DatabaseTerminal(sys.argv, self.name) self.d = terminal.get_arguments() self.xml = CommonXml() self.exc = ZmagsException(5) if self.d["database"]: self.database = Database() self.database.connect() self.products, self.no_urls = self.database.select_products(self.d["catalog_id"], self.d["product_id"]) self.database.disconnect() else: self.get_lists_from_excel() self.add_properties(self.xml) self.handle_not_provided() self.start_urls = self.products["urls"] self.total = len(self.products["urls"])
def __init__(self, *a, **kw): super(ChomeSpider, self).__init__(*a, **kw) dispatcher.connect(self.spider_closed, signals.spider_closed) terminal = DatabaseTerminal(sys.argv, self.name) self.d = terminal.get_arguments() self.xml = CommonXml() self.exc = ZmagsException(5) if self.d['database']: self.database = Database() self.database.connect() self.products, self.no_urls = self.database.select_products( self.d['catalog_id'], self.d['product_id']) self.database.disconnect() else: self.get_lists_from_excel() self.add_properties(self.xml) self.images_store = "/" + settings['IMAGES_STORE'] self.total = len(self.no_urls['product_ids'])
def __init__(self, *a, **kw): super(SportmanSpider, self).__init__(*a, **kw) dispatcher.connect(self.spider_closed, signals.spider_closed) terminal = DatabaseTerminal(sys.argv, self.name) self.d = terminal.get_arguments() self.xml = CommonXml() self.exc = ZmagsException(5, "Sportmann") if self.d["database"]: self.database = Database() self.database.connect() self.products, self.no_urls = self.database.select_products(self.d["catalog_id"], self.d["product_id"]) self.database.disconnect() else: self.get_lists_from_excel() self.add_properties(self.xml) self.start_urls = self.products["urls"] self.images_store = "/" + settings["IMAGES_STORE"] self.total = len(self.start_urls)
def __init__(self, *a, **kw): super(GuitarCenterSpider, self).__init__(*a, **kw) dispatcher.connect(self.spider_closed, signals.spider_closed) terminal = DatabaseTerminal(sys.argv, self.name) self.d = terminal.get_arguments() self.xml = CommonXml() self.exc = ZmagsException(5) if self.d['database']: self.database = Database() self.database.connect() self.products, self.no_urls = self.database.select_products(self.d['catalog_id'], self.d['product_id']) self.database.disconnect() else: self.get_lists_from_excel() self.add_properties(self.xml) self.handle_not_provided() self.start_urls = self.products['urls'] self.total = len(self.products['urls'])
def __init__(self, *a, **kw): super(BurtonSpider, self).__init__(*a, **kw) dispatcher.connect(self.spider_closed, signals.spider_closed) terminal = DatabaseTerminal(sys.argv, self.name) self.d = terminal.get_arguments() self.xml = CommonXml() self.exc = ZmagsException(5, "Burton") if self.d['database']: self.database = Database() self.database.connect() self.products, self.no_urls = self.database.select_products(self.d['catalog_id'], self.d['product_id']) self.database.disconnect() else: self.get_lists_from_excel() self.handle_not_provided() burton.add_properties(self.xml) self.start_urls = self.products['urls'] self.start_urls = ["http://www.dickssportinggoods.com/product/index.jsp?productId=13243074"] self.images_store = "/" + settings['IMAGES_STORE'] self.total = len(self.start_urls)
def __init__(self, *a, **kw): super(LydiasSpider, self).__init__(*a, **kw) dispatcher.connect(self.spider_closed, signals.spider_closed) terminal = DatabaseTerminal(sys.argv, self.name) self.d = terminal.get_arguments() self.xml = VariantsXml() self.exc = ZmagsException(5) if self.d['database']: self.database = Database() self.database.connect() self.products, self.no_urls = self.database.select_products(self.d['catalog_id'], self.d['product_id']) self.database.disconnect() else: self.get_lists_from_excel() # fix for bug with links they provide self.products['urls'] = basic.cut_string_field(self.products['urls'], "&cat=") self.handle_not_provided() self.start_urls = self.products['urls'] self.images_store = "/" + settings['IMAGES_STORE'] lydias.add_properties(self.xml) self.total = len(self.products['urls'])
def __init__(self, *a, **kw): super(BurtonSpider, self).__init__(*a, **kw) dispatcher.connect(self.spider_closed, signals.spider_closed) terminal = DatabaseTerminal(sys.argv, self.name) self.d = terminal.get_arguments() self.xml = CommonXml() self.exc = ZmagsException(5, "Burton") if self.d['database']: self.database = Database() self.database.connect() self.products, self.no_urls = self.database.select_products( self.d['catalog_id'], self.d['product_id']) self.database.disconnect() else: self.get_lists_from_excel() self.handle_not_provided() burton.add_properties(self.xml) self.start_urls = self.products['urls'] self.start_urls = [ "http://www.dickssportinggoods.com/product/index.jsp?productId=13243074" ] self.images_store = "/" + settings['IMAGES_STORE'] self.total = len(self.start_urls)
def __init__(self, *a, **kw): super(LydiasSpider, self).__init__(*a, **kw) dispatcher.connect(self.spider_closed, signals.spider_closed) terminal = DatabaseTerminal(sys.argv, self.name) self.d = terminal.get_arguments() self.xml = VariantsXml() self.exc = ZmagsException(5) if self.d['database']: self.database = Database() self.database.connect() self.products, self.no_urls = self.database.select_products( self.d['catalog_id'], self.d['product_id']) self.database.disconnect() else: self.get_lists_from_excel() # fix for bug with links they provide self.products['urls'] = basic.cut_string_field(self.products['urls'], "&cat=") self.handle_not_provided() self.start_urls = self.products['urls'] self.images_store = "/" + settings['IMAGES_STORE'] lydias.add_properties(self.xml) self.total = len(self.products['urls'])