def save_to_base(grab, task): try: slando = Slandos() title = grab.doc.select("//h1").text() if grab.doc.select('//div[@class="promo promo-richtext"]').exists(): text = grab.doc.select('//div[@class="promo promo-richtext"]').text() else: text = grab.doc.select('//div[@class="promo"]').text() deteil = [] room = '' for det in grab.doc.select('//tr'): deteil.append(det.text()) root = Selector(fromstring(det.html())) if root.select('//tr/th').exists(): if root.select('//tr/th').text() == u'Количество комнат': room = root.select('//tr/td').text() if root.select('//tr/th').text() == u'Телефон': phones = root.select('//tr/td').text().split('+')[1:] deteil = ', '.join(deteil) raw_phones = moder_phone(phones, slando) cost = grab.doc.select('//div[@class="app-price-line"]/span[@class="app-price"]').text() if title in text: text = text+deteil+room+' '+raw_phones.decode('utf-8')+' '+cost else: text = title+'. '+text+deteil+room+' '+raw_phones.decode('utf-8')+' '+cost slando.title = title slando.text = text slando.cost = cost slando.room = room slando.status = 0 slando.link = grab.response.url slando.cat = task.cat[0] slando.ci_cat = task.cat[1] slando.date = datetime.datetime.utcnow().replace(tzinfo=utc).strftime("%Y-%m-%d") slando.site = 'est.ua.final' photo = get_adv_photo(grab) slando.save() print slando.id print datetime.datetime.now().strftime("%Y-%m-%d: %H-%M") # print 'task.cat', '==', task.cat[0], '=>', task.cat[0] print grab.response.url print raw_phones.encode('utf-8'), u'----', cost.encode('utf-8'), u'----', room.encode('utf-8') return photo, slando, True except Exception, ex: print 'Error on line {}'.format(sys.exc_info()[-1].tb_lineno) # выводит строку(номер) ошибки print ex return False, False, False
def test_select_select(self): root = Selector(self.tree) self.assertEquals( set(['one', 'yet one']), set([x.text() for x in root.select('//ul').select('./li[1]')]), )
def test_text_selector(self): sel = Selector(self.tree).select('//li/text()').one() self.assertTrue(isinstance(sel, TextSelector))
def test_number(self): self.assertEquals(4, Selector(self.tree).select('//li[last()]').number()) self.assertEquals( 6, Selector(self.tree).select('//li[last()]/@id').number())
def test_textselector(self): self.assertEquals('one', Selector(self.tree).select('//li/text()').text())
def test_html(self): sel = Selector(self.tree.xpath('//h1')[0]) self.assertEquals('<h1>test</h1>', sel.html().strip())
def test_one(self): sel = Selector(self.tree).select('//ul/li') self.assertEquals('one', sel.one().node.text) self.assertEquals('one', sel.text())
def test_exists(self): sel = Selector(self.tree).select('//li[4]') self.assertEquals(True, sel.exists()) sel = Selector(self.tree).select('//li[5]') self.assertEquals(False, sel.exists())
def test_number(self): sel = Selector(self.tree).select('//li[4]') self.assertEquals(4, sel.number())
def test_attr_list(self): root = Selector(self.tree) self.assertEquals(set(['li-1', 'li-2']), set(root.select('//ul[@id="second-list"]/li')\ .attr_list('class')) )
def test_text_list(self): root = Selector(self.tree) self.assertEquals(set(['one', 'yet one']), set(root.select('//ul/li[1]').text_list()), )
def test_select_select(self): root = Selector(self.tree) self.assertEquals(set(['one', 'yet one']), set([x.text() for x in root.select('//ul').select('./li[1]')]), )
def test_text_list(self): root = Selector(self.tree) self.assertEquals( set(['one', 'yet one']), set(root.select('//ul/li[1]').text_list()), )
def test_in_general(self): sel = Selector(self.tree)
def test_select_node(self): self.assertEquals('test', Selector(self.tree).select('//h1')[0].node.text)