class TelegramNotifier: def __init__(self, chat_ids: str, username: str, module: str): if not chat_ids: logging.warning('Telegram id not set, skip initialization of telegram notifier.') self.bot = None return token = get_token('TELEGRAM_TOKEN') if not token: raise ValueError('TELEGRAM_TOKEN is null, please fill in it.') self.bot = telegram.Bot(token=token) self.chat_ids = chat_ids.split(',') self.username = username self.module = module self.sleeper = Sleeper(1) self.send_message('Init telegram bot succeed.') def send_message(self, message: str, disable_preview: bool=False): if not self.bot: logging.warning('Telegram notifier not initialized, skip.') return try: for chat_id in self.chat_ids: self.bot.send_message(chat_id=chat_id, text='[{}][{}] {}'.format(self.username, self.module, message), disable_web_page_preview=disable_preview, timeout=50) self.sleeper.sleep(normal=True) except (BadRequest, RetryAfter, TimedOut, NetworkError) as e: logging.error('Sending message error {}, retrying...'.format(e)) self.sleeper.sleep(normal=False) self.send_message(message)
class LikeMonitor: def __init__(self, username: str, telegram_chat_ids: str): self.sleeper = Sleeper(30) self.username = username self.existing_like_id_set = get_like_id_set(self.get_likes()) logging.info('Init monitor succeed.\nUsername: {}\nLike ids: {}'.format( self.username, self.existing_like_id_set)) self.telegram_notifier = TelegramNotifier(chat_ids=telegram_chat_ids, username=username, module='Like') self.last_log_time = datetime.now() def get_likes(self, max_number: int = 200) -> list: url = 'https://api.twitter.com/1.1/favorites/list.json' params = {'screen_name': self.username, 'count': max_number} json_response = send_get_request(url, params) while not json_response: self.sleeper.sleep(normal=False) json_response = send_get_request(url, params) return json_response def run(self): while True: self.sleeper.sleep(normal=True) likes = self.get_likes() for like in reversed(likes[:-10]): if like['id'] not in self.existing_like_id_set: self.telegram_notifier.send_message('@{}: {}'.format( like['user']['screen_name'], like['text'])) self.existing_like_id_set |= get_like_id_set(likes) if datetime.now() - self.last_log_time > timedelta(hours=1): logging.info('Existing like id number: {}'.format(len(self.existing_like_id_set))) self.last_log_time = datetime.now()
def __init__(self, username: str, telegram_chat_ids: str): self.sleeper = Sleeper(30) self.username = username self.existing_like_id_set = get_like_id_set(self.get_likes()) logging.info('Init monitor succeed.\nUsername: {}\nLike ids: {}'.format( self.username, self.existing_like_id_set)) self.telegram_notifier = TelegramNotifier(chat_ids=telegram_chat_ids, username=username, module='Like') self.last_log_time = datetime.now()
def __init__(self, username: str, telegram_chat_ids: str): self.sleeper = Sleeper(120) self.username = username self.user_id = get_user_id(username) self.following_users = self.get_all_following_users(self.user_id) logging.info( 'Init monitor succeed.\nUsername: {}\nUser id: {}\nFollowing users: {}' .format(username, self.user_id, self.following_users)) self.telegram_notifier = TelegramNotifier(chat_ids=telegram_chat_ids, username=username, module='Following') self.last_log_time = datetime.now()
def __init__(self, username: str, telegram_chat_ids: str): self.sleeper = Sleeper(10) self.user_id = get_user_id(username) tweets = self.get_tweets() self.last_tweet_id = tweets[0]['id'] logging.info( 'Init monitor succeed.\nUsername: {}\nUser id: {}\nLast tweet: {}'. format(username, self.user_id, tweets[0])) self.telegram_notifier = TelegramNotifier(chat_ids=telegram_chat_ids, username=username, module='Tweet') self.last_log_time = datetime.now()
def __init__(self, chat_ids: str, username: str, module: str): if not chat_ids: logging.warning('Telegram id not set, skip initialization of telegram notifier.') self.bot = None return token = get_token('TELEGRAM_TOKEN') if not token: raise ValueError('TELEGRAM_TOKEN is null, please fill in it.') self.bot = telegram.Bot(token=token) self.chat_ids = chat_ids.split(',') self.username = username self.module = module self.sleeper = Sleeper(1) self.send_message('Init telegram bot succeed.')
class TweetMonitor: def __init__(self, username: str, telegram_chat_ids: str): self.sleeper = Sleeper(10) self.user_id = get_user_id(username) tweets = self.get_tweets() self.last_tweet_id = tweets[0]['id'] logging.info( 'Init monitor succeed.\nUsername: {}\nUser id: {}\nLast tweet: {}'. format(username, self.user_id, tweets[0])) self.telegram_notifier = TelegramNotifier(chat_ids=telegram_chat_ids, username=username, module='Tweet') self.last_log_time = datetime.now() def get_tweets(self, since_id: str = None) -> list: url = 'https://api.twitter.com/2/users/{}/tweets'.format(self.user_id) params = {'max_results': 100} if since_id: params['since_id'] = since_id json_response = send_get_request(url, params) while not json_response: self.sleeper.sleep(normal=False) json_response = send_get_request(url, params) return json_response.get('data', []) def run(self): while True: self.sleeper.sleep(normal=True) tweets = self.get_tweets(since_id=self.last_tweet_id) if tweets: for tweet in tweets: self.telegram_notifier.send_message(tweet['text']) self.last_tweet_id = tweets[0]['id'] if datetime.now() - self.last_log_time > timedelta(hours=1): logging.info('Last tweet id: {}'.format(self.last_tweet_id)) self.last_log_time = datetime.now()
class FollowingMonitor: def __init__(self, username: str, telegram_chat_ids: str): self.sleeper = Sleeper(120) self.username = username self.user_id = get_user_id(username) self.following_users = self.get_all_following_users(self.user_id) logging.info( 'Init monitor succeed.\nUsername: {}\nUser id: {}\nFollowing users: {}' .format(username, self.user_id, self.following_users)) self.telegram_notifier = TelegramNotifier(chat_ids=telegram_chat_ids, username=username, module='Following') self.last_log_time = datetime.now() def get_all_following_users(self, user_id: str) -> set: url = 'https://api.twitter.com/2/users/{}/following'.format(user_id) params = {'max_results': 1000} json_response = send_get_request(url, params) while not json_response: self.sleeper.sleep(normal=False) json_response = send_get_request(url, params) results = json_response.get('data', []) next_token = json_response.get('meta', {}).get('next_token', '') while next_token: params['pagination_token'] = next_token json_response = send_get_request(url, params) while not json_response: self.sleeper.sleep(normal=False) json_response = send_get_request(url, params) results.extend(json_response.get('data', [])) next_token = json_response.get('meta', {}).get('next_token', '') return set([result.get('username', '') for result in results]) def get_user_details(self, username: str) -> str: url = 'https://api.twitter.com/2/users/by/username/{}'.format(username) params = { 'user.fields': 'name,description,url,created_at,public_metrics' } json_response = send_get_request(url, params) errors = json_response.get('errors', None) if errors: return '\n'.join([error.get('detail', '') for error in errors]) data = json_response.get('data', {}) details_str = 'Name: {}'.format(data.get('name', '')) details_str += '\nBio: {}'.format(data.get('description', '')) website = data.get('url', '') if website: details_str += '\nWebsite: {}'.format(website) details_str += '\nJoined at: {}'.format(data.get('created_at', '')) public_metrics = data.get('public_metrics', {}) details_str += '\nFollowing: {}'.format( public_metrics.get('following_count', -1)) details_str += '\nFollowers: {}'.format( public_metrics.get('followers_count', -1)) details_str += '\nTweets: {}'.format( public_metrics.get('tweet_count', -1)) if public_metrics.get('following_count', 2000) < 2000: following_users = self.get_all_following_users( get_user_id(username)) details_str += '\nFollow each other: {}'.format( 'Yes' if self.username in following_users else 'No') return details_str def detect_changes(self, old_following_users: set, new_following_users: set): if old_following_users == new_following_users: return max_changes = max(len(old_following_users) / 2, 10) if abs(len(old_following_users) - len(new_following_users)) > max_changes: return dec_users = old_following_users - new_following_users if dec_users: logging.info('Unfollow: {}'.format(dec_users)) for dec_user in dec_users: self.telegram_notifier.send_message('Unfollow: @{}\n{}'.format( dec_user, self.get_user_details(dec_user)), disable_preview=True) inc_users = new_following_users - old_following_users if inc_users: logging.info('Follow: {}'.format(inc_users)) for inc_user in inc_users: self.telegram_notifier.send_message('Follow: @{}\n{}'.format( inc_user, self.get_user_details(inc_user)), disable_preview=True) def run(self): while True: self.sleeper.sleep(normal=True) following_users = self.get_all_following_users(self.user_id) self.detect_changes(self.following_users, following_users) self.following_users = following_users if datetime.now() - self.last_log_time > timedelta(hours=1): logging.info('Number of following users: {}'.format( len(self.following_users))) self.last_log_time = datetime.now()
# Imports import torch import math import copy from sleeper import Sleeper sleeper = Sleeper() class Academy: def __init__(self, net, data, gpu = False, autoencoder_trainer = False): """ Trains and test models on datasets Args: net (Pytorch model class): Network to train and test data (Tensor Object): Data to train and test with gpu (bool, optional): If True then GPU's will be used. Defaults to False. autoencoder_trainer (bool, optional): If True labels will equal inputs. Defaults to False. """ super(Academy,self).__init__() # Declare GPU usage self.gpu = gpu # Push net to CPU or GPU self.net = net if self.gpu == False else net.cuda()
""" This script will take a model and dataset and put them in an academy to train """ # Imports from autoencoder import Autoencoder from data_setup import Data from academy import Academy import torch import os # Sleep function from sleeper import Sleeper sleeper = Sleeper() sleeper.check() # Hyperparameters csv_file = "../data/Weather/weatherdata_merged2.csv" n_epochs = 1000 gpu = False save_model = True # Push to GPU if necessary if gpu: os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = "1" # Load data number_of_unique_categories = 100 # For years data_set = Data(csv_file=csv_file, gpu=gpu,