def old_cleaner_task():
logger.info('Cleaning process started ... ')
last_date = datetime.today() - timedelta(2)
result = Article.objects.filter(date__lt=last_date).delete()
logger.info('Cleaning completed. Deleted info: {}'.format(result))
return True
def save_to_db(articles, source):
"""Save parsed articles to database"""
for article in articles:
if article and article.get('header'):
article = Article(
source=source,
url=article.get('url'),
header=article.get('header'),
picture=article.get('picture'),
text=article.get('text'),
date=article.get('date'),
)
article.save()
logger.info('Created new article {} with url: {}'.format(article.id, article.url))
def start_parsing(source=None):
"""Fires parsing process."""
if source is None:
sources = Source.objects.all()
else:
sources = Source.objects.filter(label=source)
for source in sources:
with factory.create(source.label) as parser:
if parser and parser.test_connection():
logger.info('Successfully connected to source {}'.format(source.name))
articles = parser.do_parse(source.url)
save_to_db(articles, source)
else:
logger.error('Can\'t connect to source {}!'.format(source.name))
def celery_health_checker():
logger.info('CLR_OK')
def main_parse_process_task():
logger.info('Starting parsing process ... ')
start_parsing()
logger.info('Parsing completed!')
def __exit__(self, exc_type, exc_val, exc_tb):
logger.info('exiting...')
self.driver.close()
return True # Suppressing exception to continue parsing process for next sources