def sub_monitor_check_post_old(self, submission, monitored_sub):
if self.sub_monitor.has_post_been_checked(submission.id):
log.debug('Post %s has already been checked', submission.id)
return
with self.uowm.start() as uow:
post = uow.posts.get_by_post_id(submission.id)
if not post:
log.info('Post %s does exist, sending to ingest queue',
submission.id)
post = submission_to_post(submission)
celery.send_task(
'redditrepostsleuth.core.celery.ingesttasks.save_new_post',
args=[post],
queue='postingest')
return
title_keywords = []
if monitored_sub.title_ignore_keywords:
title_keywords = monitored_sub.title_ignore_keywords.split(',')
if not self.sub_monitor.should_check_post(
post, title_keyword_filter=title_keywords):
return
self.sub_monitor.check_submission(submission, monitored_sub, post)
def save_unknown_post(self, post_id: Text) -> Optional[Post]:
"""
If we received a request on a post we haven't ingest save it
:rtype: Optional[Post]
:param post_id: Submission ID
:return: Post object
"""
submission = self.reddit.submission(post_id)
try:
post = pre_process_post(submission_to_post(submission), self.uowm,
None)
except InvalidImageUrlException:
return
except Forbidden:
log.error('Failed to download post %s, appears we are banned',
post_id)
return
if not post or post.post_type != 'image':
log.error(
'Problem ingesting post. Either failed to save or it is not an image'
)
return
return post
def save_unknown_post(post_id: str, uowm: UnitOfWorkManager,
reddit: RedditManager) -> Post:
"""
If we received a request on a post we haven't ingest save it
:param submission: Reddit Submission
:return:
"""
submission = reddit.submission(post_id)
post = pre_process_post(submission_to_post(submission), uowm, None)
if not post or post.post_type != 'image':
log.error(
'Problem ingesting post. Either failed to save or it is not an image'
)
return
return post
def ingest_new_posts(self):
while True:
sr = self.reddit.subreddit('all')
try:
while True:
try:
for submission in sr.stream.submissions():
log.debug('Saving post %s', submission.id)
post = submission_to_post(submission)
if not post.post_type:
post.post_type = post_type_from_url(post.url)
log.error('Last resort post type %s',
post.post_type)
log.error(post.url)
save_new_post.apply_async((post, ),
queue='postingest')
except Forbidden as e:
pass
except Exception as e:
log.exception('INGEST THREAD DIED', exc_info=True)
def save_unknown_post(self, post_id: str) -> Post:
"""
If we received a request on a post we haven't ingest save it
:param submission: Reddit Submission
:return:
"""
log.info('Post %s does not exist, attempting to ingest', post_id)
submission = self.reddit.submission(post_id)
post = None
try:
post = pre_process_post(submission_to_post(submission), self.uowm,
None)
except InvalidImageUrlException:
log.error('Failed to ingest post %s. URL appears to be bad',
post_id)
if not post:
log.error(
'Problem ingesting post. Either failed to save or it is not an image'
)
return
return post
def ingest_without_stream(self):
seen_posts = []
while True:
try:
if len(seen_posts) > 10000:
seen_posts = []
try:
submissions = [
sub
for sub in self.reddit.subreddit('all').new(limit=500)
]
except ResponseException as e:
if e.response.status_code == 429:
log.error('Too many requests from IP. Waiting')
time.sleep(60)
continue
except Exception as e:
if 'code: 429' in str(e):
log.error('Too many requests from IP. Waiting')
time.sleep(60)
continue
log.debug('%s posts from API', len(submissions))
for submission in submissions:
if submission.id in seen_posts:
continue
#log.debug('Saving post %s', submission.id)
post = submission_to_post(submission)
if not post.post_type:
post.post_type = post_type_from_url(post.url)
#log.debug('Last resort post type %s', post.post_type)
#log.debug(post.url)
save_new_post.apply_async((post, ), queue='postingest')
seen_posts.append(post.post_id)
except Exception as e:
log.exception('INGEST THREAD DIED', exc_info=True)