def update_top_stories(self):
try:
stories = self.client.get_front_page_stories()
for data in stories:
story_data = data['data']
story, created = Story.objects.get_or_create(service=self.service, code=story_data.get('permalink'))
if created:
story.date = timezone.datetime.fromtimestamp(story_data.get('created_utc'), timezone.get_current_timezone())
story.build_url()
score = story_data.get('score', 0)
comments = story_data.get('num_comments', 0)
has_changes = (score != story.score or comments != story.comments)
if not story.status == Story.NEW and has_changes:
update = StoryUpdate(story=story)
update.comments_changes = comments - story.comments
update.score_changes = score - story.score
update.save()
story.comments = comments
story.score = score
story.title = story_data.get('title', '')
story.nsfw = story_data.get('over_18', False)
story.status = Story.OK
story.save()
except Exception, e:
logging.error(e)
def update_top_stories(self):
try:
posts = self.client.get_top_stories()
today = timezone.now()
for post_data in posts:
story, created = Story.objects.get_or_create(service=self.service, code=post_data['id'], date=timezone.datetime(today.year, today.month, today.day, tzinfo=timezone.get_current_timezone()))
if created:
story.url = u'{0}/@{1}/{2}'.format(self.service.story_url, post_data['creator']['username'], post_data['id'])
story.start_score = int(post_data['virtuals']['recommends'])
story.start_comments = int(post_data['virtuals']['responsesCreatedCount'])
story.title = post_data['title']
recommends = int(post_data['virtuals']['recommends']) - story.start_score
comments = int(post_data['virtuals']['responsesCreatedCount']) - story.start_comments
has_changes = (recommends != story.score or comments != story.comments)
if not story.status == Story.NEW and has_changes:
update = StoryUpdate(story=story)
update.comments_changes = comments - story.comments
update.score_changes = recommends - story.score
update.save()
story.score = recommends
story.comments = comments
story.status = Story.OK
story.save()
except Exception, e:
logging.error(e)
def update_story(self, code):
try:
story_data = self.client.get_story(code)
if story_data and story_data['type'] == 'story':
story, created = Story.objects.get_or_create(
service=self.service, code=code)
if story_data.get('deleted', False):
story.delete()
return
if story.status == Story.NEW:
story.date = timezone.datetime.fromtimestamp(
story_data.get('time'),
timezone.get_current_timezone())
story.url = u'{0}{1}'.format(story.service.story_url,
story.code)
score = story_data.get('score', 0)
comments = story_data.get('descendants', 0)
has_changes = (score != story.score
or comments != story.comments)
if not story.status == Story.NEW and has_changes:
update = StoryUpdate(story=story)
update.comments_changes = comments - story.comments
update.score_changes = score - story.score
update.save()
story.comments = comments
story.score = score
story.title = story_data.get('title', '')
url = story_data.get('url', '')
if url:
story.content_type = Story.URL
story.content = url
text = story_data.get('text', '')
if text:
story.content_type = Story.TEXT
story.content = text
story.status = Story.OK
story.save()
except Exception, e:
logging.error(
u'Exception in code {0} HackerNewsCrawler.update_story'.format(
code))
logging.error(e)
def update_story(self, code):
try:
story_data = self.client.get_story(code)
if story_data and story_data['type'] == 'story':
story, created = Story.objects.get_or_create(service=self.service, code=code)
if story_data.get('deleted', False):
story.delete()
return
if story.status == Story.NEW:
story.date = timezone.datetime.fromtimestamp(story_data.get('time'), timezone.get_current_timezone())
story.url = u'{0}{1}'.format(story.service.story_url, story.code)
score = story_data.get('score', 0)
comments = story_data.get('descendants', 0)
has_changes = (score != story.score or comments != story.comments)
if not story.status == Story.NEW and has_changes:
update = StoryUpdate(story=story)
update.comments_changes = comments - story.comments
update.score_changes = score - story.score
update.save()
story.comments = comments
story.score = score
story.title = story_data.get('title', '')
url = story_data.get('url', '')
if url:
story.content_type = Story.URL
story.content = url
text = story_data.get('text', '')
if text:
story.content_type = Story.TEXT
story.content = text
story.status = Story.OK
story.save()
except Exception, e:
logging.error(u'Exception in code {0} HackerNewsCrawler.update_story'.format(code))
logging.error(e)
