def save_to_base(grab, task):
try:
slando = Slandos()
title = grab.doc.select("//h1").text()
if grab.doc.select('//div[@class="promo promo-richtext"]').exists():
text = grab.doc.select('//div[@class="promo promo-richtext"]').text()
else:
text = grab.doc.select('//div[@class="promo"]').text()
deteil = []
room = ''
for det in grab.doc.select('//tr'):
deteil.append(det.text())
root = Selector(fromstring(det.html()))
if root.select('//tr/th').exists():
if root.select('//tr/th').text() == u'Количество комнат':
room = root.select('//tr/td').text()
if root.select('//tr/th').text() == u'Телефон':
phones = root.select('//tr/td').text().split('+')[1:]
deteil = ', '.join(deteil)
raw_phones = moder_phone(phones, slando)
cost = grab.doc.select('//div[@class="app-price-line"]/span[@class="app-price"]').text()
if title in text:
text = text+deteil+room+' '+raw_phones.decode('utf-8')+' '+cost
else:
text = title+'. '+text+deteil+room+' '+raw_phones.decode('utf-8')+' '+cost
slando.title = title
slando.text = text
slando.cost = cost
slando.room = room
slando.status = 0
slando.link = grab.response.url
slando.cat = task.cat[0]
slando.ci_cat = task.cat[1]
slando.date = datetime.datetime.utcnow().replace(tzinfo=utc).strftime("%Y-%m-%d")
slando.site = 'est.ua.final'
photo = get_adv_photo(grab)
slando.save()
print slando.id
print datetime.datetime.now().strftime("%Y-%m-%d: %H-%M")
# print 'task.cat', '==', task.cat[0], '=>', task.cat[0]
print grab.response.url
print raw_phones.encode('utf-8'), u'----', cost.encode('utf-8'), u'----', room.encode('utf-8')
return photo, slando, True
except Exception, ex:
print 'Error on line {}'.format(sys.exc_info()[-1].tb_lineno) # выводит строку(номер) ошибки
print ex
return False, False, False
def test_select_select(self):
root = Selector(self.tree)
self.assertEquals(
set(['one', 'yet one']),
set([x.text() for x in root.select('//ul').select('./li[1]')]),
)
def test_text_selector(self):
sel = Selector(self.tree).select('//li/text()').one()
self.assertTrue(isinstance(sel, TextSelector))
def test_number(self):
self.assertEquals(4,
Selector(self.tree).select('//li[last()]').number())
self.assertEquals(
6,
Selector(self.tree).select('//li[last()]/@id').number())
def test_textselector(self):
self.assertEquals('one',
Selector(self.tree).select('//li/text()').text())
def test_html(self):
sel = Selector(self.tree.xpath('//h1')[0])
self.assertEquals('<h1>test</h1>', sel.html().strip())
def test_html(self):
sel = Selector(self.tree.xpath('//h1')[0])
self.assertEquals('<h1>test</h1>', sel.html().strip())
def test_one(self):
sel = Selector(self.tree).select('//ul/li')
self.assertEquals('one', sel.one().node.text)
self.assertEquals('one', sel.text())
def test_exists(self):
sel = Selector(self.tree).select('//li[4]')
self.assertEquals(True, sel.exists())
sel = Selector(self.tree).select('//li[5]')
self.assertEquals(False, sel.exists())
def test_exists(self):
sel = Selector(self.tree).select('//li[4]')
self.assertEquals(True, sel.exists())
sel = Selector(self.tree).select('//li[5]')
self.assertEquals(False, sel.exists())
def test_number(self):
sel = Selector(self.tree).select('//li[4]')
self.assertEquals(4, sel.number())
def test_one(self):
sel = Selector(self.tree).select('//ul/li')
self.assertEquals('one', sel.one().node.text)
self.assertEquals('one', sel.text())
def test_attr_list(self):
root = Selector(self.tree)
self.assertEquals(set(['li-1', 'li-2']),
set(root.select('//ul[@id="second-list"]/li')\
.attr_list('class'))
)
def test_text_list(self):
root = Selector(self.tree)
self.assertEquals(set(['one', 'yet one']),
set(root.select('//ul/li[1]').text_list()),
)
def test_select_select(self):
root = Selector(self.tree)
self.assertEquals(set(['one', 'yet one']),
set([x.text() for x in root.select('//ul').select('./li[1]')]),
)
def test_text_list(self):
root = Selector(self.tree)
self.assertEquals(
set(['one', 'yet one']),
set(root.select('//ul/li[1]').text_list()),
)
def test_attr_list(self):
root = Selector(self.tree)
self.assertEquals(set(['li-1', 'li-2']),
set(root.select('//ul[@id="second-list"]/li')\
.attr_list('class'))
)
def test_in_general(self):
sel = Selector(self.tree)
def test_number(self):
sel = Selector(self.tree).select('//li[4]')
self.assertEquals(4, sel.number())
def test_select_node(self):
self.assertEquals('test',
Selector(self.tree).select('//h1')[0].node.text)
