def process_segment(id):
with closing(database.postgresql['session']()) as session:
segment = session.query(models.segment).get(id)
if not segment:
return
if segment.status:
return
profile = models.profile()
profile = get_graph(profile, segment.get_url())
profile.urls_local = get_response(
'https://www.facebook.com/%(value)s' % {
'value': segment.value,
}).url.replace('http://', 'https://')
if profile.urls_local:
urls = profile.get_urls()
if urls:
profile = get_www(profile, urls['www'])
profile = get_www_info(profile, urls['www_info'])
profile = get_www_likes(profile, urls['www_likes'])
session.add(profile)
session.commit()
session.refresh(profile)
profile.refresh()
session.add(profile)
session.commit()
segment.status = True
session.add(segment)
session.commit()
def process_result(id):
with closing(database.postgresql['session']()) as session:
result = session.query(models.result).get(id)
if not result:
return
if result.status:
return
contents = get_response(result.get_url()).text
if 'Security Check' in contents:
raise Exception('Security Check')
selector = Selector(text=contents)
for td in selector.xpath(
'//table[@class="uiGrid _51mz mam"]/tbody/tr/td'):
name = None
try:
name = td.xpath('div/a/@title').extract()[0]
except IndexError:
pass
if not name:
raise Exception('Invalid Name')
image = None
try:
image = td.xpath('div/a/img/@src').extract()[0]
except IndexError:
pass
if not image:
raise Exception('Invalid Image')
categories_primary = None
try:
categories_primary = td.xpath(
'div/div/div/div/div/text()').extract()[0]
except IndexError:
pass
urls_local = None
try:
urls_local = td.xpath('div/a/@href').extract()[0].replace(
'http://', 'https://')
except IndexError:
pass
if not urls_local:
raise Exception('Invalid URL')
profile = session.query(models.profile, ).filter(
models.profile.urls_local == urls_local, ).first()
if not profile:
profile = models.profile()
profile.name = name
profile.image = image
profile.categories_primary = categories_primary
profile.urls_local = urls_local
try:
session.add(profile)
session.commit()
except SQLAlchemyError:
session.rollback()
raise
result.status = True
session.add(result)
session.commit()
