def test_os_environ(mock_os):
expected_value = 'environment_value'
mock_os.environ.__contains__.return_value = True # patch in statement
mock_os.environ.__getitem__.return_value = expected_value
os_variable = env('a_key')
assert expected_value == os_variable
mock_os.environ.__getitem__.assert_called_with('a_key')
from ingestor.readers.read_tweet_json import read_tweet_json
from ingestor.senders.send_event_hub_message import send_event_hub_message, stop_client
from ingestor.utils.env import env
from ingestor.utils.logging_service import LoggingService
logger = LoggingService('Send tweet.json').logger
PROFILE = env('PROFILE_NAME')
message_type = 'tweet'
data_delimiter = '|~|'
def build_message(profile: str,
message_type: str,
event_id: str,
message: str,
*,
delimiter: str = data_delimiter):
structured_message = \
f'profile:{profile}{delimiter}' \
f'type:{message_type}{delimiter}' \
f'id:{event_id}{delimiter}' \
f'message:{message}'
return structured_message
if __name__ == "__main__":
tweets = read_tweet_json()
from ingestor.builders.build_firebase import get_firebase_auth_and_db
from ingestor.utils.env import env
USER = env('FIREBASE_AUTH_USER')
PASSWORD = env('FIREBASE_AUTH_PASSWORD')
def build_firebase_connection_for_tests():
auth, db = get_firebase_auth_and_db()
user = auth.sign_in_with_email_and_password(USER, PASSWORD)
id_token = user['idToken']
return db, id_token
import time
from azure.eventhub import EventHubClient, Offset
from ingestor.builders.build_firebase import get_firebase_auth_and_db
from ingestor.daos.last_received_event_offset import LastReceivedEventOffset
from ingestor.retrievers.event_hub_retriever import EventHubRetriever
from ingestor.utils.env import env
from ingestor.utils.logging_service import LoggingService
logger = LoggingService('Receive').logger
CONNECTION_STRING = env('EVENT_HUB_CONNECTION_STRING')
EVENT_HUB_TOPIC = env('EVENT_HUB_TOPIC_HELLO_WORLD_NAME')
EVENT_HUB_PARTITION = env('EVENT_HUB_TOPIC_HELLO_WORLD_PARTITION')
CONSUMER_GROUP = "$default"
PREFETCH = 1
OFFSET = Offset('1', inclusive=True)
USER = env('FIREBASE_AUTH_USER')
PASSWORD = env('FIREBASE_AUTH_PASSWORD')
def receive_one():
auth, db = get_firebase_auth_and_db()
user = auth.sign_in_with_email_and_password(USER, PASSWORD)
id_token = user['idToken']
last_received_event_offset = LastReceivedEventOffset(db, id_token)
import logging
import sys
from ingestor.utils.env import env
default_level = env('DEFAULT_LOGGING_LEVEL', default=logging.INFO)
default_logging_format = env('DEFAULT_LOGGING_FORMAT',
default='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
class LoggingService:
def __init__(self, name: str, *, logging_format: str = default_logging_format, level: int = default_level):
self.name = name
self.logging_format = logging_format
self.level = level
self._logger = None
@property
def logger(self):
if self._logger:
return self._logger
self._logger = self._build_logger()
return self._logger
def info(self, message):
self.logger.info(message)
def warning(self, message):
self.logger.warning(message)
def test_os_environ_key_missing_with_default():
expected_value = 'a_default'
os_variable = env('a_key', default=expected_value)
assert expected_value == os_variable
def test_os_environ_key_missing():
expected_value = None
os_variable = env('a_key')
assert expected_value == os_variable
from collections import defaultdict
from ingestor.utils.env import env
from ingestor.utils.logging_service import LoggingService
logger = LoggingService('extract_hashtags_from_tweets_list').logger
profile = env('PROFILE_NAME')
def extract_hashtags(tweets: list) -> dict:
hashtags_found = defaultdict(int)
for tweet in tweets:
tweet_hashtags = tweet['entities']['hashtags']
[_add_to_aggregate(hashtags_found, tweet_hashtag['text'].lower()) for tweet_hashtag in tweet_hashtags]
return hashtags_found
def _add_to_aggregate(aggregate_dict: dict, key: str):
aggregate_dict[key] = aggregate_dict[key] + 1
from azure.eventhub import EventHubClient
from ingestor.utils.env import env
from ingestor.utils.logging_service import LoggingService
logger = LoggingService('build_sender_client').logger
CONNECTION_STRING = env('EVENT_HUB_CONNECTION_STRING')
EVENT_HUB_TOPIC = env('EVENT_HUB_TOPIC_HELLO_WORLD_NAME')
EVENT_HUB_PARTITION = env('EVENT_HUB_TOPIC_HELLO_WORLD_PARTITION')
def build_sender_client(*,
topic: str = EVENT_HUB_TOPIC,
partition: str = EVENT_HUB_PARTITION) -> tuple:
if not CONNECTION_STRING:
raise ValueError("No EventHubs URL supplied.")
client = EventHubClient.from_connection_string(CONNECTION_STRING, topic)
sender = client.add_sender(partition)
client.run()
return client, sender
from app.harvesters.twitter.api import get_api
from app.harvesters.twitter_harvester import TwitterHarvester
from ingestor.utils.env import env
env("")
twitter_harvester = TwitterHarvester(get_api(), 'dev3l_', '')
twitter_harvester.fetch()