def test_log(self):
sio = io.StringIO()
handler = logging.StreamHandler(stream=sio)
logger = log.Logger(handlers=[handler])
logger.set_module_level("base.test_log", log.Level.INFO)
logger.info("Fo{x}", x="o") # == Foo
logger.debug("Bar")
handler.close()
sio.seek(0)
stmt = sio.read()
self.assertIn(
"/base/test_log.py:21 [base.test_log] test_log() : Foo\n", stmt)
self.assertIn(" INFO ", stmt)
import base.singleton as sn
import base.log as l
import google.cloud as gl_cloud
import google.cloud.language as gl_lang
import data_providers.external_apis.google_nlp_enum_string as maps
logger = l.Logger('GoogleNlpApi', None)
# todo catch exceptions or pass retry object to retry calls at least 1 time
class GoogleNlpApi(sn.Singleton):
def __init__(self):
self.emotions_api = GoogleNlpApiEmotions.get_instance()
self.meaning_api = GoogleNlpApiMeaning.get_instance()
def query_emotions(self, *args, **kwargs):
return self.emotions_api.query_emotions(*args, **kwargs)
def query_meaning(self, *args, **kwargs):
return self.meaning_api.query_meaning(*args, **kwargs)
class GoogleNlpApiEmotions(sn.Singleton):
def __init__(self):
self.client = gl_cloud.language.LanguageServiceClient()
def query_emotions(self, text, by_sentence):
document = gl_lang.types.Document(
import base.singleton as sn
import base.log as l
import data_providers.external_apis as e_a
import requests
import json
logger = l.Logger('TextsummarizationNetApi')
class TextsummarizationNetApi(sn.Singleton):
def __init__(self):
self.api_url = 'https://textanalysis-text-summarization.p.mashape.com/text-summarizer'
self.sentences_count = 7
def summarise_url(self, url):
try:
return self.unsafe_summarise_url(url)
except Exception as e:
logger.log(' exception {}'.format(e))
return None
def unsafe_summarise_url(self, url):
params = {'url': url, 'text': None, 'sentnum': self.sentences_count}
headers = {
'X-Mashape-Key': open(e_a.summariser_key_path, 'r').read(),
'Content-Type': 'application/json',
'Accept': 'application/json'
}
response = requests.post(self.api_url,
json.dumps(params),
import config as conf
import base.singleton as sn
import business_logic.data_tags as tags
import json
import re
import base.log as l
logger = l.Logger('PatternBasedExtractor')
class PatternBasedExtractor(sn.Singleton):
patterns_keys = [
tags.SubType.news, tags.SubType.social_media, tags.SubType.stock,
tags.SubType.industry
]
patterns = {
tags.SubType.news: ['news', 'information', 'headlines'],
tags.SubType.social_media: ['think', 'talk', 'social'],
tags.SubType.stock: [
'price', 'much', 'stock', 'variance', 'behaviour', 'volatility',
'perform', 'rise', 'fall', 'behave', 'move', 'change'
],
tags.SubType.industry: ['industry', 'sector']
}
indicators_for_behaviour = [
tags.Indicator.stock_volatility, tags.Indicator.price_change,
tags.Indicator.just_price
]
import base.singleton as sn
import business_logic.nlp.pattern_based_extractor as pbe
import data_providers.external_apis.google_nlp_api as google_nlp
import business_logic.nlp.nlp_exceptions as ex
import base.log as l
import business_logic.data_tags as tags
import config as conf
import google.cloud.language as gl_lang
import multiprocessing.pool as m_pool
import re
logger = l.Logger('GoogleCommandExtractor', None)
class GoogleCommandExtractor(sn.Singleton):
def __init__(self):
self.pattern_based_extractor: pbe.PatternBasedExtractor = pbe.PatternBasedExtractor.get_instance(
)
self.google_api = google_nlp.GoogleNlpApi.get_instance()
self.interesting_parts_of_speech = [
gl_lang.enums.PartOfSpeech.Tag.NOUN,
gl_lang.enums.PartOfSpeech.Tag.ADJ,
gl_lang.enums.PartOfSpeech.Tag.NUM
]
def get_meaning_from_alternatives(self, alternatives):
if len(alternatives) == 0:
return None
import base.singleton as sn
from data_providers.database_connections import mysql_key_path
import base.log as l
import config
import pymysql
import json
logger = l.Logger('MySqlConnection', max_len=None)
class MySqlConnection():
def __init__(self):
self.conn = None
self.open()
def open(self):
self.conn = pymysql.connect(host=config.mysql_host,
user='******',
password='******',
db='cs261')
def execute(self, sql, data):
logger.log(('executing {}'.format(sql)))
many = type(data) is list
if not self.conn.open:
self.open()
try:
with self.conn.cursor() as cursor:
if many:
import base.log as l
import base.singleton as sn
import business_logic.speech_recognition.text_audio as text_audio
import business_logic.data_tags as tags
logger = l.Logger('MessageWorker')
class MessageWorker(sn.Singleton):
def set_server(self, server):
self.server = server
def set_message_router(self, message_router):
self.message_router = message_router
def __init__(self):
self.server = None
self.message_router = None
self.text_audio: text_audio.TextAudio = text_audio.TextAudio.get_instance(
)
def on_message(self, message):
logger.log('from client: {}'.format(message))
message_body = message['body']
if message_body['mime_type'] == 'text/plain':
self.on_text(message_body)
elif 'audio' == message_body['mime_type'].split('/')[0]:
self.on_audio(message_body)
else:
self.send(self.on_unknown_mime_response())
import base.singleton as sn
import base.log as l
import config
import google.cloud as gl_cloud
import google.cloud.speech as gl_speech
logger = l.Logger('GoogleSpeechRecognitionApi')
class GoogleSpeechRecognitionApi(sn.Singleton):
def __init__(self):
self.client = gl_cloud.speech.SpeechClient()
def query(self,
audio_bytes_dict,
n_alternatives=config.default_number_of_speech_rec_alternatives):
google_config = gl_speech.types.RecognitionConfig(
encoding=gl_speech.enums.RecognitionConfig.AudioEncoding.FLAC,
language_code='en-GB',
max_alternatives=n_alternatives)
content = audio_bytes_dict['bytes']
audio = gl_speech.types.RecognitionAudio(content=content)
response = self.client.recognize(google_config, audio)
if len(response.results) == 0:
return []
alternatives_list = [{
'text': a.transcript,
import base.singleton as sn
import business_logic.data_tags as tags
import business_logic.data_processing.social_media_analyser as sm_analyser
import business_logic.data_processing.news_analyser as news_analyser
import data_providers.data_wrappers.sql_database_wrapper as sql_connection
import business_logic.data_processing.indicators as indicators
import base.log as l
logger = l.Logger("WorldData")
class WorldData(sn.Singleton):
def __init__(self):
self.news_analyser = news_analyser.NewsAnalyser.get_instance()
self.social_media_analyser = sm_analyser.SocialMediaAnalyser.get_instance(
)
self.sql_wrapper: sql_connection.SqlDatabaseWrapper = sql_connection.SqlDatabaseWrapper.get_instance(
)
self.indicators = indicators.Indicator.get_instance()
def get_news(self, json_request):
return self.news_analyser.get_news(json_request)
def get_public_opinion(self, json_request):
return self.social_media_analyser.get_public_opinion(json_request)
def get_indicator(self, request):
if "time" not in request.keys():
request["time"] = tags.TimePeriods.default_time_period
start_time, end_time = request["time"].to_interval()
import base.log as l
import base.singleton as sn
import sumy.parsers.html as s_html
import sumy.nlp.tokenizers as s_tokenizers
import sumy.nlp.stemmers as s_stemmers
import sumy.utils as s_utils
from sumy.summarizers.luhn import LuhnSummarizer as Summarizer
logger = l.Logger('SummarisationWithSumy')
class SummarisationWithSumy(sn.Singleton):
def __init__(self):
self.language = 'english'
self.sentences_count = 7
self.stemmer = s_stemmers.Stemmer(self.language)
self.summarizer = Summarizer(self.stemmer)
self.summarizer.stop_words = s_utils.get_stop_words(self.language)
def summarise_url(self, url):
try:
return self.unsafe_summarise_url(url)
except Exception as e:
logger.log(' exception {}'.format(e))
return None
def unsafe_summarise_url(self, url):
import base.singleton as sn
import pydub
import tempfile
import io
import base64
import business_logic.data_tags as tags
import base.log as l
logger = l.Logger('FlacConverter')
class FlacConverter(sn.Singleton):
def __init__(self):
pass
def to_flac(self, audio_bytes_dict):
given_mime = audio_bytes_dict['mime_type']
ext = given_mime.split('/')[1]
given_bytes = audio_bytes_dict['bytes']
with tempfile.NamedTemporaryFile(
suffix='.' + ext) as temp_file, io.BytesIO() as flac_bytes:
temp_file.write(given_bytes)
audio_segment = pydub.AudioSegment.from_file(temp_file.name,
ext).set_channels(1)
should_update = audio_segment.channels != 1 or given_bytes != tags.MimeTypes.flac
if audio_segment.channels != 1:
audio_segment = pydub.AudioSegment.from_file(
import base.singleton as sn
import business_logic.message_routing as routing
import config
import base.log as l
import os
import base.news_page_handler as news_page_handler
import tornado.template as template
logger = l.Logger('HtmlGenerator')
class HtmlGenerator(sn.Singleton):
def __init__(self):
self.loader = template.Loader(config.templates_folder)
self.news_template = self.loader.load('news_template.html')
@staticmethod
def static_url(filename):
return os.path.join(config.static_folder, filename)
def make_page_for_news(self, news, request):
logger.log('creating page for {}'.format(request))
with open(routing.news_page_path, 'wb') as new_page:
page = self.news_template.generate(
news=news,
keywords=request['keywords'],
static_url=HtmlGenerator.static_url)
new_page.write(page)
news_page_handler.NewsPageHandler.news_page = page
import base.log as l
import base.singleton as sn
import data_providers.external_apis.twitter_api as twitter_api
logger = l.Logger('SocialMediaProvider')
class SocialMediaProvider(sn.Singleton):
def __init__(self):
self.social_media_apis = [
twitter_api.TwitterApi.get_instance(),
# potentially add more
]
def get_posts_from_given_media_by_keywords(self, media_provider, keywords):
return media_provider.get_posts_by_keywords(keywords)
def get_posts_by_keywords(self, keywords):
posts = []
for api in self.social_media_apis:
from_this_api = self.get_posts_from_given_media_by_keywords(
api, keywords)
logger.log('got {} posts from {} for keywords {}'.format(
len(from_this_api), str(api.name), keywords))
posts.extend(from_this_api)
return posts
import base.singleton as sn
import base.log as l
import data_providers.data_wrappers.social_media_provider as sm_provider
import business_logic.nlp.nlp as nlp
import business_logic.data_tags as tags
import multiprocessing.pool as m_pool
import numpy as np
logger = l.Logger('SocialMediaAnalyser', None)
class SocialMediaAnalyser(sn.Singleton):
def __init__(self):
self.sm_provider = sm_provider.SocialMediaProvider.get_instance()
self.nlp = nlp.NLP.get_instance()
self.threshold_0 = 0.15
self.threshold_1 = 0.45
def get_public_opinion(self, request_dict):
keywords = request_dict['keywords']
posts_likes = self.sm_provider.get_posts_by_keywords(keywords)
pool = m_pool.ThreadPool(processes=len(posts_likes))
async_result = pool.map_async(self.nlp.get_emotions_score,
[p['text'] for p in posts_likes])
emotions = np.array(async_result.get())
return self.analytics(emotions, keywords)
def analytics(self, emotions, keywords):
import base.singleton as sn
import base.log as l
import business_logic.nlp.nlp as nlp
import business_logic.data_tags as tags
import business_logic.message_routing.readable_responser as rr
import business_logic.data_processing.world_data as world_data
import business_logic.data_processing.my_data as my_data
import business_logic.message_routing.html_generator as html_generator
import base.message_worker as mw
import business_logic.notifications.adviser as adviser
logger = l.Logger('MessageRouter')
class MessageRouter(sn.Singleton):
def set_message_worker(self, message_worker):
self.message_worker = message_worker
def __init__(self):
self.nlp = nlp.NLP.get_instance()
self.readable_responser: rr.ReadableResponser = rr.ReadableResponser.get_instance(
)
self.world_data: world_data.WorldData = world_data.WorldData.get_instance(
)
self.my_data: my_data.MyData = my_data.MyData.get_instance()
self.html_generator: html_generator.HtmlGenerator = html_generator.HtmlGenerator.get_instance(
)
self.message_worker: mw.MessageWorker = None
self.adviser: adviser.Adviser = adviser.Adviser.get_instance()
def process_alternatives(self, alternatives):
return str(o)
if isinstance(o, tags.TimePeriods.Month) \
or isinstance(o, tags.TimePeriods.Week) \
or isinstance(o, tags.TimePeriods.Day) \
or isinstance(o, tags.TimePeriods.Hour) \
or isinstance(o, tags.TimePeriods.RightNow):
interval = o.to_interval()
interval[0] = interval[0].isoformat()
interval[1] = interval[1].isoformat()
return interval
default_old = json.JSONEncoder.default
json.JSONEncoder.default = default_with_dates
handler_logger = l.Logger('Handler')
class Handler(tws.WebSocketHandler):
on_message_callback = None
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
handler_logger.log(self, 'connected')
Server.get_instance().add_handler(self)
def open(self):
pass
def on_message(self, message):
handler_logger.log(' received {}'.format(message))
