import datetime import sqlite3 import requests import time import json from requests_oauthlib import OAuth1 from tweepy import Stream, OAuthHandler from tweepy.streaming import StreamListener from log import log_config logger = log_config.getLogger('twitter_mod.py') class SnetListener(StreamListener): """ Extended Tweet Listener """ def __init__(self, msg_limit=0, time_limit=0): logger.debug("SnetListener INIT") self.start_time = time.time() self.time_limit = time_limit self.msg_limit = msg_limit self.msg_counter = 0 self.sentences = [] self.status_error_code = None super(SnetListener, self).__init__() def on_data(self, data): """Called when raw data is received from connection.
import pathlib import subprocess import threading import time import sys import argparse from services import registry from log import log_config logger = log_config.getLogger('run_service.py') def main(): logger.debug('call => main()') parser = argparse.ArgumentParser(description="Run services") parser.add_argument("--no-daemon", action="store_false", dest="run_daemon", help="do not start the daemon") parser.add_argument("--daemon-config-path", help="Path to daemon configuration file", required=False) args = parser.parse_args() root_path = pathlib.Path(__file__).absolute().parent # All services modules go here service_modules = ['services.named_entity_recognition'] # Call for all the services listed in service_modules start_all_services(root_path, service_modules, args.run_daemon,
import path_setup import grpc from services.service_spec import sentiment_analysis_rpc_pb2_grpc as grpc_services from services.service_spec import sentiment_analysis_rpc_pb2 as rpc from test_data import test_sentences from services import registry from log import log_config logger = log_config.getLogger('test_service.py', test=True) channel = None if __name__ == '__main__': try: logger.debug('call => __name == __main__') # Service ONE - Sentiment Analysis endpoint = 'localhost:{}'.format( registry['sentiment_analysis']['grpc']) # Open a gRPC channel channel = grpc.insecure_channel('{}'.format(endpoint)) except KeyError as e: print(e) try: logger.debug("call => SentimentAnalysis() Service Test Starting... ") # SentimentAnalysis() Method Test # create a stub (client) stub = grpc_services.SentimentAnalysisStub(channel) # create a valid request message
import path_setup import base64 import compile_proto from services import named_entity_recognition as ner from test_data import b64_sentences from log import log_config logger = log_config.getLogger('run_unit_test.py', test=True) class Request(object): def __init__(self): self.value = "" def test_compiled(): assert compile_proto.success def test_recognize(): """ Test Named Entity Recognition :return: """ servicer = ner.RecognizeMessageServicer() request = Request() request.value = b64_sentences.senteces() context = object() response = servicer.Recognize(request, context) decoded_result = base64.b64decode(response.value).decode('utf-8')
import base64 import grpc import concurrent.futures as futures from services.modules import consensus_mod, twitter_mod from services.service_spec import sentiment_analysis_rpc_pb2_grpc as grpc_services from services.service_spec.sentiment_analysis_rpc_pb2 import OutputMessage from services import common from log import log_config from nltk.sentiment import SentimentIntensityAnalyzer # Services Path current_path = os.path.dirname(os.path.realpath(__file__)) parent_path = os.path.abspath(os.path.join(current_path, os.pardir)) service_root_path = os.path.abspath(os.path.join(parent_path, os.pardir)) logger = log_config.getLogger('sentiment_analysis.py') class ShowMessageServicer(grpc_services.ShowMessageServicer): """ Create a class to be added to the gRPC server derived from the protobuf codes. """ def __init__(self): logger.debug("call => ShowMessageServicer()") def Show(self, request, context): """ The method that will be exposed to the snet-cli call command. :param request: incoming data :param context: object that provides RPC-specific information (timeout, etc). :return:
import sys import grpc import base64 import concurrent.futures as futures from services.modules import entity_recognizer_mod from services.service_spec import named_entity_recognition_rpc_pb2_grpc as grpc_bt_grpc from services.service_spec.named_entity_recognition_rpc_pb2 import OutputMessage from services import common from log import log_config logger = log_config.getLogger('named_entity_recognition.py') # Create a class to be added to the gRPC server # derived from the protobuf codes. class ShowMessageServicer(grpc_bt_grpc.ShowMessageServicer): def __init__(self): # Just for debugging purpose. logger.debug("ShowMessageServicer created") # The method that will be exposed to the snet-cli call command. # request: incoming data # context: object that provides RPC-specific information (timeout, etc). def Show(self, request, context): # In our case, request is a InputMessage() object (from .proto file) self.value = request.value # To respond we need to create a OutputMessage() object (from .proto file) self.result = OutputMessage() self.result.value = "Processed => " + self.value
import path_setup import grpc from services.service_spec import named_entity_recognition_rpc_pb2_grpc as grpc_bt_grpc from services.service_spec import named_entity_recognition_rpc_pb2 as grpc_bt_pb2 from services import registry from test_data import test_sentences from log import log_config logger = log_config.getLogger('test_service.py') channel = None if __name__ == '__main__': try: logger.debug('call => __name == __main__') logger.debug("call => Creating channel() Starting... ") endpoint = 'localhost:{}'.format( registry['named_entity_recognition']['grpc']) # Open a gRPC channel channel = grpc.insecure_channel('{}'.format(endpoint)) except Exception as e: logger.debug("Error found Creating Channel => " + e) try: logger.debug("call => RecognizeMessage() Method Test Starting... ") # RecognizeMessage() Method Test # create a stub (client) stub = grpc_bt_grpc.RecognizeMessageStub(channel) # create a valid request message test_data = test_sentences.senteces()
import os from log import log_config import nltk from nltk import pos_tag from nltk.tag import StanfordNERTagger from nltk.tokenize import word_tokenize from nltk.chunk import conlltags2tree from nltk.tree import Tree logger = log_config.getLogger('entity_recognizer_mod.py') #Service paths current_path = os.path.dirname(os.path.realpath(__file__)) parent_path = os.path.abspath(os.path.join(current_path, os.pardir)) service_root_path = os.path.abspath(os.path.join(parent_path, os.pardir)) # Snet Classifier class SnetEntityRecognizer: def __init__(self): # Snet Classifier. logger.debug("SnetEntityRecognizer INIT") self.english_model = service_root_path + '/models/stanford-ner-2018-02-27/classifiers/english.all.3class.distsim.crf.ser.gz' self.stanford_jar = service_root_path + '/models/stanford-ner-2018-02-27/stanford-ner-3.9.1.jar' # Process text def process_text(self, input_text): token_text = word_tokenize(str(input_text)) return token_text # Stanford NER tagger
import os import pickle from nltk.classify import ClassifierI from statistics import mode from nltk.tokenize import word_tokenize from log import log_config logger = log_config.getLogger('consensus_mod.py') #Service paths current_path = os.path.dirname(os.path.realpath(__file__)) parent_path = os.path.abspath(os.path.join(current_path, os.pardir)) service_root_path = os.path.abspath(os.path.join(parent_path, os.pardir)) class VoteClassifier(ClassifierI): """ Vote by classifiers results """ def __init__(self, *classifiers): self._classifiers = classifiers def classify(self, features): """ Vote on all classifiers results :param features: incomming feature to be classified :return: vote winner """ votes = [] for c in self._classifiers: v = c.classify(features) votes.append(v)
import os import pickle from nltk.classify import ClassifierI from statistics import mode from nltk.tokenize import word_tokenize from log import log_config logger = log_config.getLogger('analyze_mod.py') #Service paths current_path = os.path.dirname(os.path.realpath(__file__)) parent_path = os.path.abspath(os.path.join(current_path, os.pardir)) service_root_path = os.path.abspath(os.path.join(parent_path, os.pardir)) class VoteClassifier(ClassifierI): """ Vote by classifiers results """ def __init__(self, *classifiers): self._classifiers = classifiers def classify(self, features): """ Vote on all classifiers results :param features: incomming feature to be classified :return: vote winner """ votes = [] for c in self._classifiers: v = c.classify(features) votes.append(v)