def process_segment(id): with closing(database.postgresql['session']()) as session: segment = session.query(models.segment).get(id) if not segment: return if segment.status: return profile = models.profile() profile = get_graph(profile, segment.get_url()) profile.urls_local = get_response( 'https://www.facebook.com/%(value)s' % { 'value': segment.value, }).url.replace('http://', 'https://') if profile.urls_local: urls = profile.get_urls() if urls: profile = get_www(profile, urls['www']) profile = get_www_info(profile, urls['www_info']) profile = get_www_likes(profile, urls['www_likes']) session.add(profile) session.commit() session.refresh(profile) profile.refresh() session.add(profile) session.commit() segment.status = True session.add(segment) session.commit()
def process_result(id): with closing(database.postgresql['session']()) as session: result = session.query(models.result).get(id) if not result: return if result.status: return contents = get_response(result.get_url()).text if 'Security Check' in contents: raise Exception('Security Check') selector = Selector(text=contents) for td in selector.xpath( '//table[@class="uiGrid _51mz mam"]/tbody/tr/td'): name = None try: name = td.xpath('div/a/@title').extract()[0] except IndexError: pass if not name: raise Exception('Invalid Name') image = None try: image = td.xpath('div/a/img/@src').extract()[0] except IndexError: pass if not image: raise Exception('Invalid Image') categories_primary = None try: categories_primary = td.xpath( 'div/div/div/div/div/text()').extract()[0] except IndexError: pass urls_local = None try: urls_local = td.xpath('div/a/@href').extract()[0].replace( 'http://', 'https://') except IndexError: pass if not urls_local: raise Exception('Invalid URL') profile = session.query(models.profile, ).filter( models.profile.urls_local == urls_local, ).first() if not profile: profile = models.profile() profile.name = name profile.image = image profile.categories_primary = categories_primary profile.urls_local = urls_local try: session.add(profile) session.commit() except SQLAlchemyError: session.rollback() raise result.status = True session.add(result) session.commit()