def get_connection(self) -> MongoClient:
try:
conn = MongoClient(self.uri, serverSelectionTimeoutMS=1)
conn.server_info()
try:
conn.admin.command("ismaster")
except ConnectionFailure:
logger.error(f"Could not connect to MongoDB: {error}")
finally:
try:
conn["admin"].command("serverStatus")
except Exception as error:
logger.error(f"General Exception Mongo - {error}")
else:
logger.info("You are connected!")
return conn
except ServerSelectionTimeoutError as error:
logger.error(f"Server Timeout Error - {error}")
def consumer() -> NoReturn:
logger.info(f"This is {realtime}...")
consumer = TwitterConsumer(broker, tweets_topic).consumer
if consumer.bootstrap_connected():
logger.info("Connection okay.")
for messagem in consumer:
logger.info(messagem.value)
else:
logger.error("Failed to connect to service")
def get(self):
logger.info("GET - covid tweets...")
tweets = tweet_clean.filter_tweets(Twitter().search(**query_tweets))
mongo_client = mongo.get_connection()
db = mongo_client["twitter"]
collection = db["covid"]
if tweets:
logger.info("200 - GET - successfully get covid tweets")
for tweet in tweets:
try:
found = collection.find({
"_id": dict(tweet)["_id"]
}).limit(1).count()
if found > 0:
logger.info(
"Data alredy exist in MongoDB. Continue...")
continue
else:
insert = collection.insert_one(tweet)
if insert.inserted_id:
logger.info(
"Insert data covid tweet in MongoDB - Successfully insert data!"
)
else:
logger.error(
"Insert data covid tweet in MongoDB - Bad insert data..."
)
except DuplicateKeyError as error:
logger.error("400 - GET - no covid tweets")
logger.error(f"DuplicateKeyError - {error}")
return {"message": responses[400], "count": 0}, 400
return {"message": responses[200], "count": len(tweets)}, 200
else:
logger.error("400 - GET - no covid tweets")
return {"message": responses[400], "count": 0}, 400
mongo.close_connection(mongo_client)
def list_databases(self) -> NoReturn:
for index, value in enumerate(self.client.list_database_names(), start=1):
logger.info(f"Database {index} - {value}")
def close_connection(self, client: Callable) -> NoReturn:
logger.info("Connection getting closed")
client.close()
def get(self):
logger.info("GET - sentimental analysis of covid tweets...")
mongo_client = mongo.get_connection()
db = mongo_client["twitter"]
collection_covid = db["covid"]
collection_sentimental = db["sentimental"]
if collection_sentimental.count() == 0:
logger.info("Sentimental collection is empty")
if "sentimental" in db.list_collection_names():
logger.info("Sentimental collection exist")
else:
logger.info("Sentimental collection not exist")
else:
logger.info("Sentimental collection is not empty")
try:
tweets = [[
tweet["_id"], tweet["full_text"]
] for tweet in [tweet for tweet in collection_covid.find()]]
if len(tweets) == 0:
logger.warning("No covid tweets. Empty information")
except Exception as error:
logger.error("400 - GET - {}".format(error))
return {"message": str(error), "count": 0}, 400
if tweets:
sentiments = analyzer.get_tweets_sentiment(tweets, sentiment=None)
if sentiments:
logger.info(
"200 - GET - successfully get sentimental analysis of covid tweets"
)
neg = analyzer.get_tweets_sentiment(tweets, sentiment="neg")
pos = analyzer.get_tweets_sentiment(tweets, sentiment="pos")
neutros = analyzer.get_tweets_sentiment(tweets,
sentiment="neutros")
for tweet in sentiments:
try:
found = collection_sentimental.find({
"tweet_id":
dict(tweet)["tweet_id"]
}).limit(1).count()
if found > 0:
logger.info(
"Data alredy exist in MongoDB. Continue...")
continue
else:
insert = collection_sentimental.insert_one(tweet)
if insert.inserted_id:
logger.info(
"Insert data sentimental analysis in MongoDB - Successfully insert data!"
)
else:
logger.error(
"Insert data sentimental analysis in MongoDB - Bad insert data..."
)
except pymongo.errors.DuplicateKeyError as error:
logger.error(
"400 - GET - no data sentimental analysis of covid tweets"
)
logger.error(f"DuplicateKeyError - {error}")
return {"message": responses[400], "count": 0}, 400
return {
"message": responses[200],
"count": len(sentiments)
}, 200
else:
logger.error(
"400 - GET - no data sentimental analysis of covid tweets")
return {
"message": responses[400],
"count": 0,
}, 400
else:
logger.info("400 - GET - no covid tweets")
return {
"message": responses[400],
"data": [],
"count": 0,
}, 400
mongo.close_connection(mongo_client)
def on_connect(self) -> NoReturn:
logger.info("You are now connected to twitter streaming API.")