def old_cleaner_task(): logger.info('Cleaning process started ... ') last_date = datetime.today() - timedelta(2) result = Article.objects.filter(date__lt=last_date).delete() logger.info('Cleaning completed. Deleted info: {}'.format(result)) return True
def save_to_db(articles, source): """Save parsed articles to database""" for article in articles: if article and article.get('header'): article = Article( source=source, url=article.get('url'), header=article.get('header'), picture=article.get('picture'), text=article.get('text'), date=article.get('date'), ) article.save() logger.info('Created new article {} with url: {}'.format(article.id, article.url))
def start_parsing(source=None): """Fires parsing process.""" if source is None: sources = Source.objects.all() else: sources = Source.objects.filter(label=source) for source in sources: with factory.create(source.label) as parser: if parser and parser.test_connection(): logger.info('Successfully connected to source {}'.format(source.name)) articles = parser.do_parse(source.url) save_to_db(articles, source) else: logger.error('Can\'t connect to source {}!'.format(source.name))
def celery_health_checker(): logger.info('CLR_OK')
def main_parse_process_task(): logger.info('Starting parsing process ... ') start_parsing() logger.info('Parsing completed!')
def __exit__(self, exc_type, exc_val, exc_tb): logger.info('exiting...') self.driver.close() return True # Suppressing exception to continue parsing process for next sources