class TelegramNotifier:
def __init__(self, chat_ids: str, username: str, module: str):
if not chat_ids:
logging.warning('Telegram id not set, skip initialization of telegram notifier.')
self.bot = None
return
token = get_token('TELEGRAM_TOKEN')
if not token:
raise ValueError('TELEGRAM_TOKEN is null, please fill in it.')
self.bot = telegram.Bot(token=token)
self.chat_ids = chat_ids.split(',')
self.username = username
self.module = module
self.sleeper = Sleeper(1)
self.send_message('Init telegram bot succeed.')
def send_message(self, message: str, disable_preview: bool=False):
if not self.bot:
logging.warning('Telegram notifier not initialized, skip.')
return
try:
for chat_id in self.chat_ids:
self.bot.send_message(chat_id=chat_id,
text='[{}][{}] {}'.format(self.username, self.module, message),
disable_web_page_preview=disable_preview,
timeout=50)
self.sleeper.sleep(normal=True)
except (BadRequest, RetryAfter, TimedOut, NetworkError) as e:
logging.error('Sending message error {}, retrying...'.format(e))
self.sleeper.sleep(normal=False)
self.send_message(message)
class LikeMonitor:
def __init__(self, username: str, telegram_chat_ids: str):
self.sleeper = Sleeper(30)
self.username = username
self.existing_like_id_set = get_like_id_set(self.get_likes())
logging.info('Init monitor succeed.\nUsername: {}\nLike ids: {}'.format(
self.username, self.existing_like_id_set))
self.telegram_notifier = TelegramNotifier(chat_ids=telegram_chat_ids,
username=username,
module='Like')
self.last_log_time = datetime.now()
def get_likes(self, max_number: int = 200) -> list:
url = 'https://api.twitter.com/1.1/favorites/list.json'
params = {'screen_name': self.username, 'count': max_number}
json_response = send_get_request(url, params)
while not json_response:
self.sleeper.sleep(normal=False)
json_response = send_get_request(url, params)
return json_response
def run(self):
while True:
self.sleeper.sleep(normal=True)
likes = self.get_likes()
for like in reversed(likes[:-10]):
if like['id'] not in self.existing_like_id_set:
self.telegram_notifier.send_message('@{}: {}'.format(
like['user']['screen_name'], like['text']))
self.existing_like_id_set |= get_like_id_set(likes)
if datetime.now() - self.last_log_time > timedelta(hours=1):
logging.info('Existing like id number: {}'.format(len(self.existing_like_id_set)))
self.last_log_time = datetime.now()
def __init__(self, username: str, telegram_chat_ids: str):
self.sleeper = Sleeper(30)
self.username = username
self.existing_like_id_set = get_like_id_set(self.get_likes())
logging.info('Init monitor succeed.\nUsername: {}\nLike ids: {}'.format(
self.username, self.existing_like_id_set))
self.telegram_notifier = TelegramNotifier(chat_ids=telegram_chat_ids,
username=username,
module='Like')
self.last_log_time = datetime.now()
def __init__(self, username: str, telegram_chat_ids: str):
self.sleeper = Sleeper(120)
self.username = username
self.user_id = get_user_id(username)
self.following_users = self.get_all_following_users(self.user_id)
logging.info(
'Init monitor succeed.\nUsername: {}\nUser id: {}\nFollowing users: {}'
.format(username, self.user_id, self.following_users))
self.telegram_notifier = TelegramNotifier(chat_ids=telegram_chat_ids,
username=username,
module='Following')
self.last_log_time = datetime.now()
def __init__(self, username: str, telegram_chat_ids: str):
self.sleeper = Sleeper(10)
self.user_id = get_user_id(username)
tweets = self.get_tweets()
self.last_tweet_id = tweets[0]['id']
logging.info(
'Init monitor succeed.\nUsername: {}\nUser id: {}\nLast tweet: {}'.
format(username, self.user_id, tweets[0]))
self.telegram_notifier = TelegramNotifier(chat_ids=telegram_chat_ids,
username=username,
module='Tweet')
self.last_log_time = datetime.now()
def __init__(self, chat_ids: str, username: str, module: str):
if not chat_ids:
logging.warning('Telegram id not set, skip initialization of telegram notifier.')
self.bot = None
return
token = get_token('TELEGRAM_TOKEN')
if not token:
raise ValueError('TELEGRAM_TOKEN is null, please fill in it.')
self.bot = telegram.Bot(token=token)
self.chat_ids = chat_ids.split(',')
self.username = username
self.module = module
self.sleeper = Sleeper(1)
self.send_message('Init telegram bot succeed.')
class TweetMonitor:
def __init__(self, username: str, telegram_chat_ids: str):
self.sleeper = Sleeper(10)
self.user_id = get_user_id(username)
tweets = self.get_tweets()
self.last_tweet_id = tweets[0]['id']
logging.info(
'Init monitor succeed.\nUsername: {}\nUser id: {}\nLast tweet: {}'.
format(username, self.user_id, tweets[0]))
self.telegram_notifier = TelegramNotifier(chat_ids=telegram_chat_ids,
username=username,
module='Tweet')
self.last_log_time = datetime.now()
def get_tweets(self, since_id: str = None) -> list:
url = 'https://api.twitter.com/2/users/{}/tweets'.format(self.user_id)
params = {'max_results': 100}
if since_id:
params['since_id'] = since_id
json_response = send_get_request(url, params)
while not json_response:
self.sleeper.sleep(normal=False)
json_response = send_get_request(url, params)
return json_response.get('data', [])
def run(self):
while True:
self.sleeper.sleep(normal=True)
tweets = self.get_tweets(since_id=self.last_tweet_id)
if tweets:
for tweet in tweets:
self.telegram_notifier.send_message(tweet['text'])
self.last_tweet_id = tweets[0]['id']
if datetime.now() - self.last_log_time > timedelta(hours=1):
logging.info('Last tweet id: {}'.format(self.last_tweet_id))
self.last_log_time = datetime.now()
class FollowingMonitor:
def __init__(self, username: str, telegram_chat_ids: str):
self.sleeper = Sleeper(120)
self.username = username
self.user_id = get_user_id(username)
self.following_users = self.get_all_following_users(self.user_id)
logging.info(
'Init monitor succeed.\nUsername: {}\nUser id: {}\nFollowing users: {}'
.format(username, self.user_id, self.following_users))
self.telegram_notifier = TelegramNotifier(chat_ids=telegram_chat_ids,
username=username,
module='Following')
self.last_log_time = datetime.now()
def get_all_following_users(self, user_id: str) -> set:
url = 'https://api.twitter.com/2/users/{}/following'.format(user_id)
params = {'max_results': 1000}
json_response = send_get_request(url, params)
while not json_response:
self.sleeper.sleep(normal=False)
json_response = send_get_request(url, params)
results = json_response.get('data', [])
next_token = json_response.get('meta', {}).get('next_token', '')
while next_token:
params['pagination_token'] = next_token
json_response = send_get_request(url, params)
while not json_response:
self.sleeper.sleep(normal=False)
json_response = send_get_request(url, params)
results.extend(json_response.get('data', []))
next_token = json_response.get('meta', {}).get('next_token', '')
return set([result.get('username', '') for result in results])
def get_user_details(self, username: str) -> str:
url = 'https://api.twitter.com/2/users/by/username/{}'.format(username)
params = {
'user.fields': 'name,description,url,created_at,public_metrics'
}
json_response = send_get_request(url, params)
errors = json_response.get('errors', None)
if errors:
return '\n'.join([error.get('detail', '') for error in errors])
data = json_response.get('data', {})
details_str = 'Name: {}'.format(data.get('name', ''))
details_str += '\nBio: {}'.format(data.get('description', ''))
website = data.get('url', '')
if website:
details_str += '\nWebsite: {}'.format(website)
details_str += '\nJoined at: {}'.format(data.get('created_at', ''))
public_metrics = data.get('public_metrics', {})
details_str += '\nFollowing: {}'.format(
public_metrics.get('following_count', -1))
details_str += '\nFollowers: {}'.format(
public_metrics.get('followers_count', -1))
details_str += '\nTweets: {}'.format(
public_metrics.get('tweet_count', -1))
if public_metrics.get('following_count', 2000) < 2000:
following_users = self.get_all_following_users(
get_user_id(username))
details_str += '\nFollow each other: {}'.format(
'Yes' if self.username in following_users else 'No')
return details_str
def detect_changes(self, old_following_users: set,
new_following_users: set):
if old_following_users == new_following_users:
return
max_changes = max(len(old_following_users) / 2, 10)
if abs(len(old_following_users) -
len(new_following_users)) > max_changes:
return
dec_users = old_following_users - new_following_users
if dec_users:
logging.info('Unfollow: {}'.format(dec_users))
for dec_user in dec_users:
self.telegram_notifier.send_message('Unfollow: @{}\n{}'.format(
dec_user, self.get_user_details(dec_user)),
disable_preview=True)
inc_users = new_following_users - old_following_users
if inc_users:
logging.info('Follow: {}'.format(inc_users))
for inc_user in inc_users:
self.telegram_notifier.send_message('Follow: @{}\n{}'.format(
inc_user, self.get_user_details(inc_user)),
disable_preview=True)
def run(self):
while True:
self.sleeper.sleep(normal=True)
following_users = self.get_all_following_users(self.user_id)
self.detect_changes(self.following_users, following_users)
self.following_users = following_users
if datetime.now() - self.last_log_time > timedelta(hours=1):
logging.info('Number of following users: {}'.format(
len(self.following_users)))
self.last_log_time = datetime.now()
# Imports
import torch
import math
import copy
from sleeper import Sleeper
sleeper = Sleeper()
class Academy:
def __init__(self,
net,
data,
gpu = False,
autoencoder_trainer = False):
"""
Trains and test models on datasets
Args:
net (Pytorch model class): Network to train and test
data (Tensor Object): Data to train and test with
gpu (bool, optional): If True then GPU's will be used. Defaults to False.
autoencoder_trainer (bool, optional): If True labels will equal inputs. Defaults to False.
"""
super(Academy,self).__init__()
# Declare GPU usage
self.gpu = gpu
# Push net to CPU or GPU
self.net = net if self.gpu == False else net.cuda()
"""
This script will take a model and dataset and put them in an academy to train
"""
# Imports
from autoencoder import Autoencoder
from data_setup import Data
from academy import Academy
import torch
import os
# Sleep function
from sleeper import Sleeper
sleeper = Sleeper()
sleeper.check()
# Hyperparameters
csv_file = "../data/Weather/weatherdata_merged2.csv"
n_epochs = 1000
gpu = False
save_model = True
# Push to GPU if necessary
if gpu:
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
# Load data
number_of_unique_categories = 100 # For years
data_set = Data(csv_file=csv_file,
gpu=gpu,