def process_articles(campaign, limit=100): """Test processing and DB insertion. :param campaign: :param limit: :return: """ ranked_articles = get_and_rank_news_articles() sorted_articles = sort_articles(ranked_articles) if not isinstance(limit, int) and limit > 1: raise ValueError('Invalid limit') for order, article in enumerate(sorted_articles[:limit], 1): logging.info(article) DbHelper.update_ranked_post(news_id=article.news_id, rank_score=article.score, rank_order=order) logging.info('Process %d articles', order)
def terminate(self, status=1): """Updates campaign status and sends email report if reporting was enabled. :param status: :return: """ if self.send_report: logging.info('Sending Email report') self.report.email_recipients = self.email_recipients self.report.send() if self.campaign_reference: sqlquery = 'UPDATE campaign SET campaign_end=\'%s\', status=%d ' \ 'WHERE campaign.reference=\'%s\'' % ( settings.DBNOW, status, self.campaign_reference) DbHelper.update_database(sqlquery)
def person_top(request): """ :param request: :return: """ sqlquery = """SELECT COUNT(id) AS mentions,name FROM persons GROUP BY name ORDER BY mentions DESC;""" return DbHelper.get_multiple_records(sqlquery)
def get_articles_by_date(date): """ :param date: :return: """ logging.info('Looking news for %s', date) articles = DbHelper.get_multiple_records( settings.CLUSTERING_QUERY_GET_NEWS % date) logging.info('Total news found: %s', len(articles)) return articles
def person_list(limit=10): """ :param limit: :return: """ if limit < 1: limit = 10 sqlquery = """SELECT COUNT(id) AS mentions,name FROM persons WHERE valid=True GROUP BY name ORDER BY mentions DESC LIMIT %s;""" % str(limit) return DbHelper.get_multiple_records(sqlquery)
def user_list(limit=10): """ :param limit: :return: """ if limit < 1: limit = 10 sqlquery = """SELECT id, username, created FROM api_users GROUP BY id, username, created ORDER BY created DESC LIMIT %s;""" % str(limit) return DbHelper.get_multiple_records(sqlquery)
def translate_article(campaign_instance, article, new_article, news_id): """ :param campaign_instance: :param article: :param new_article: :param report: :param news_id: :return: """ # Perform translation using Google Translate API log.info('Translating...%r', article.url) translated_text = translate_content(article.title, campaign_instance.translation_lang) if new_article and translated_text: # Update database record sql_query = update_db_query(translated_text.replace("'", "''"), settings.DEFAULT_LANGUAGE, news_id) DbHelper.update_database(sql_query) else: log.warning('Article already exists, skipping DB update') return translated_text
def get_articles_by_date(date): """ :param date: :return: list(List of RankedArticle) """ logging.info('Looking news for %s', date) news = DbHelper.get_multiple_records( settings.RANKING_QUERY_GET_NEWS_BY_DATE % date) if news: logging.info('Total news found: %s', len(news)) return news else: raise ValueError('No news found')
def person_filter(request): """ :param request: :return: """ persons = None name_value = request.args.get('name') if name_value and len(name_value) > 1: name_value = name_value.replace("'", "''") sqlquery = """SELECT COUNT(id) AS mentions,name FROM persons WHERE name='%s' AND valid=True GROUP BY name ORDER BY mentions DESC;""" % \ name_value persons = DbHelper.get_multiple_records(sqlquery) return persons
def get_articles(self, date='latest'): """Read posts from database. :param date: :return: """ # Get latest date for News inserted in Database. if date == 'latest': date = DbHelper.get_record(settings.CLUSTERING_QUERY_DATE) logging.info('Using latest date. The latest date found was: %s', date) logging.info('Using date: %s', date) if re.match('\d{4}-\d{1,2}-\d{2}', date): self.articles = get_articles_by_date(date) logging.info('Found %d articles.' % len(self.articles)) return self.articles
def process_entities(article, news_id, db_update=True): """ :param article: :param news_id: :param db_update: :return: """ os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = settings.CREDENTIALS log.info('process_entities() Processing content for : %s using NLP', article.url) num_of_entities = 0 entities = nlp.analyze_entities( '%r %r' % (article.title, article.description)) if entities: num_of_entities = len(entities) if num_of_entities < 1: log.error('No NLP entities found') return log.info('Processing %d entities: %s', num_of_entities, article.url) # Extract tags and associate them with original article. log.info('Processing article tags: %s', article.url) tags = nlp_utils.extract_tags(entities) log.info('Found %d tags', len(tags)) log.info('Processing persons in tags: %s', article.url) persons = nlp_utils.extract_entity(entities, 'PERSON') log.info('Found %d persons', len(persons)) log.info('Processing organizations in tags: %s', article.url) organizations = nlp_utils.extract_entity(entities, 'ORGANIZATION') log.info('Found %d organizations', len(organizations)) if db_update: log.info('Updating database...') for tag in tags: tag_id = DbHelper.insert_tag(tag_name=tag) if news_id and tag_id: DbHelper.associate_tag_news(news_id, tag_id) for person in persons: DbHelper.insert_person(person) for organization in organizations: DbHelper.insert_company(organization) return {'persons': persons, 'organizations': organizations}
def process_articles(articles, news_provider, campaign_instance): """Get content from News API. For each article verifies if exists in DB or not. If exists, ignore it, otherwise, process each field. Perform sentiment analysis on article content or description fields. :param articles: :param news_provider: :param campaign_instance: :return: """ news_id = None translated_text = None tweets = [] num_of_articles = len(articles) campaign_instance.set_articles(num_of_articles) report = Report.get_report(subject='News ML | %s' % news_provider) log.info('Analyzing %d articles...', num_of_articles) if num_of_articles < 1: if campaign_instance.send_report: log.warning('Skipping report via email...') log.error('No titles found') return # Create Report instance and attach recipients. log.info('Translation enabled: %s', campaign_instance.translation_enable) log.info('Email reporting enabled: %s', campaign_instance.send_report) log.info('Twitter enabled: %s', campaign_instance.twitter) log.info('Twitter image extraction: %s', settings.EXTRACT_TWITTER_IMAGE) log.info('Twitter add hash tags: %s', settings.TWITTER_ADD_HASHTAGS) if campaign_instance.send_report: report.email_recipients = campaign_instance.email_recipients for _, article in articles.items(): new_article = False if not article.description: log.error('Not description found in article: %s', article.url) continue if settings.EXTRACT_TWITTER_IMAGE: # meta:twitter:image article.twitter_image = twitter_utils.get_twitter_element( article.url, 'twitter:image') log.info('Article: %s, [%s], [%s]', article.title, article.url, article.twitter_image) if not DbHelper.record_exists(article.url): news_id = None log.info('New Article retrieved: %r, %r' % (article.title, article.url)) try: log.info('Processing sentiment analysis') score, magnitude = nlp_utils.get_sentiment_scores( article.content or article.description) log.info('Insert article into Database') news_id = DbHelper.insert_news( title=article.title, author=article.author, description=article.description, content=article.content, url=article.url, url_to_image=article.url_to_image, source_id=article.source_id, source=article.source, campaign=campaign_instance.reference, published_at=article.published_at, score=score, magnitude=magnitude, sentiment=nlp_utils.get_sentiment(score)) if not news_id: log.error('Unable to insert record %s', article.url) continue except (ValueError, UnicodeDecodeError) as exception: log.exception(exception) new_article = True if settings.PROCESS_ENTITIES: entities = common_utils.process_entities( article, news_id, True) else: log.warning('Article %r already exists ', article.url) if settings.PROCESS_ENTITIES: entities = common_utils.process_entities( article, news_id, False) if campaign_instance.translation_enable: translated_text = translate_utils.translate_article( campaign_instance, article, new_article, news_id) if len(translated_text) > 1: log.info('Adding translated content to report.') article.title = translated_text else: logging.error('Translated text is empty.') if campaign_instance.send_report: # Only send today articles in Report. today = datetime.now().date() published_at = datetime.strptime(article.published_at[:10], '%Y-%m-%d').date() if settings.REPORT_ALL_DATES_ARTICLES: log.info('Publishing all dates articles') log.info('Today: %s Report date: %s. ', today, published_at) if today == published_at or settings.REPORT_ALL_DATES_ARTICLES: log.info('Adding article information to Report: %s %s' % (article.title, article.url)) report.add_content(article.url, article.title, article.twitter_image) else: log.warning( 'Article published date is not today (%s), ' 'skipping article from Report', published_at) # Handle Twitter if campaign_instance.twitter: tweet_text = article.title if campaign_instance.translation_enable: tweet_text = translated_text if settings.TWITTER_ADD_HASHTAGS: # TODO (gogasca) Find Twitter handlers tweet_text = twitter_utils.add_hash_tags(tweet_text, entities) tweets.append('{} {}'.format(tweet_text, article.url)) if campaign_instance.send_report: log.info('Sending report via email...') report.send() if campaign_instance.twitter: log.info('Sending Tweets') twitter_utils.send_tweets(tweets, campaign_instance.twitter_delay) log.info('Extraction completed')
def launch(campaign_instance=None): """ The logic is as follows: 1. Extract Articles from <Provider: (Techmeme, Techcrunch)> web page 2. For each article extract title, short url and content. 3. Translate Article information 4. Use Google NLP to extract meaningful keywords from content 5. Insert record in database. 6. Send report. :param campaign_instance: :return: """ entities = None news_id = None num_of_articles = 0 translated_text = None tweets = [] report = Report.get_report(subject=settings.TECHMEME_REPORT) """Create Report instance and attach recipients.""" log.info('Translation enabled: %s', campaign_instance.translation_enable) log.info('Email reporting enabled: %s', campaign_instance.send_report) log.info('Twitter enabled: %s', campaign_instance.twitter) log.info('Twitter image extraction: %s', settings.EXTRACT_TWITTER_IMAGE) log.info('Twitter add hash tags: %s', settings.TWITTER_ADD_HASHTAGS) articles = extract_articles(settings.TECHMEME_URL) if articles: num_of_articles = len(articles) else: logging.error('No articles found.') log.info('Retrieving %d articles...', num_of_articles) if campaign_instance.limit > 0: logging.warning('Limit is defined. Skipping other news') articles = dict( itertools.islice(articles.items(), campaign_instance.limit)) num_of_articles = len(articles) if num_of_articles < 1: log.error('No articles found') if campaign_instance.send_report: log.warning('Skipping report via email...') return log.info('Processing %d articles...', num_of_articles) campaign_instance.set_articles(num_of_articles) if campaign_instance.send_report: report.email_recipients = campaign_instance.email_recipients for _, article in articles.items(): new_article = False if not article.title: log.warning('No title found. Article won\'t be inserted') continue if settings.EXTRACT_TWITTER_IMAGE: # meta:twitter:image article.twitter_image = twitter_utils.get_twitter_element( article.url, 'twitter:image') log.info('Article: %s, [%s], [%s]', article.title, article.url, article.twitter_image) if not DbHelper.record_exists(article.url): news_id = None log.info('New Article retrieved: %r, %r' % ( article.title, article.url)) try: log.info('Processing sentiment analysis') score, magnitude = nlp_utils.get_sentiment_scores( article.content) source = url_extract.get_domain(article.url) or '' log.info('Insert article into Database') news_id = DbHelper.insert_news(title=article.title, content=article.content, url=article.url, provider=settings.TECHMEME, source=source.upper(), source_id=source, campaign=campaign_instance.reference, score=score, magnitude=magnitude, sentiment=nlp_utils.get_sentiment( score) ) if not news_id: log.error('Unable to insert record %s', article.url) continue except (ValueError, UnicodeDecodeError) as e: log.exception(e) new_article = True if settings.PROCESS_ENTITIES: entities = common_utils.process_entities(article, news_id, True) else: log.warning('Article already exists.') if settings.PROCESS_ENTITIES: entities = common_utils.process_entities(article, news_id, False) if campaign_instance.translation_enable: translated_text = translate_utils.translate_article( campaign_instance, article, new_article, news_id) if translated_text: log.info('Adding translated content to report.') article.title = translated_text if campaign_instance.send_report: # Only send articles created 'today' in Report. today = datetime.now().date() published_at = datetime.strptime(article.published_at, '%y%m%d').date() if settings.REPORT_ALL_DATES_ARTICLES: log.info('Publishing all dates articles') log.info('Today: %s Report date: %s. ', today, published_at) if today == published_at or settings.REPORT_ALL_DATES_ARTICLES: log.info( 'Adding article information to Report: %s %s' % ( article.title, article.url)) report.add_content(article.url, article.title, article.twitter_image) else: log.warning( 'Article published date is not today (%s), ' 'skipping article from Report', published_at) if campaign_instance.twitter: tweet_text = article.title if campaign_instance.translation_enable: tweet_text = translated_text if settings.TWITTER_ADD_HASHTAGS: # TODO (gogasca) Find Twitter handlers tweet_text = twitter_utils.add_hash_tags(tweet_text, entities) tweets.append('{} {}'.format(tweet_text, article.url)) if campaign_instance.send_report: log.info('Sending email notification...') report.send() if campaign_instance.twitter: log.info('Sending Tweets...') twitter_utils.send_tweets(tweets, campaign_instance.twitter_delay) log.info('Extraction completed')
def main(): DbHelper.test_connection()