def run(self, bot):
self.logger.debug("Fetching tweets...")
tweet_rows = []
# fetch the tg users' home timelines
tg_chats = list(
TelegramChat.select().where(TelegramChat.twitter_secret != None))
for tg_chat in tg_chats:
bot_auth = bot.tw.auth
tw_api = tg_chat.tw_api(bot_auth.consumer_key,
bot_auth.consumer_secret)
try:
if tg_chat.last_tweet_id == 0:
# get just the 5 latest tweets
self.logger.debug("Fetching 5 latest tweets for {}".format(
tg_chat.chat_id))
tweets = tw_api.home_timeline(count=5,
tweet_mode='extended')
else:
# get the fresh tweets
self.logger.debug("Fetching new tweets from {}".format(
tg_chat.last_tweet_id))
tweets = tw_api.home_timeline(
since_id=tg_chat.last_tweet_id, tweet_mode='extended')
except tweepy.errors.TweepyException as e:
sc = e.response.status_code
if sc == 429:
self.logger.debug("- Hit ratelimit, breaking.")
break
self.logger.debug(
"- Unknown exception, Status code {}".format(sc))
continue
sorted_tweets = sorted(tweets, key=lambda tweet: tweet.id)
for tweet in sorted_tweets:
try:
self.logger.debug("- Got tweet #{} @{}: {}".format(
tweet.id, tweet.author.screen_name, tweet.full_text))
t = parse_tweet(tweet)
bot.send_tweet(tg_chat, t)
except Exception as e:
self.logger.error(
"- An error occurred while processing tweet #{}: {}".
format(tweet.id, e))
# save the latest tweet sent to chat
self.logger.debug("- Setting id: {}".format(tweet.id))
tg_chat.last_tweet_id = tweet.id
tg_chat.save()
self.logger.debug("Cleaning up TelegramChats marked for deletion")
for chat in TelegramChat.select().where(
TelegramChat.delete_soon == True):
chat.delete_instance(recursive=True)
self.logger.debug("Deleting chat {}".format(chat.chat_id))
def handle_chat(update: telegram.Update, context: CallbackContext) -> None:
bot = context.bot
chat, _created = TelegramChat.get_or_create(
chat_id=update.message.chat.id,
tg_type=update.message.chat.type,
)
bot.reply(update, "Hey! Use commands to talk with me, please! See /help")
def run(self, bot):
self.logger.debug("Fetching tweets...")
tg_chats = list(TelegramChat.select().where(
TelegramChat.twitter_secret is not None))
for tg_chat in tg_chats:
self.subscription_changed(bot, tg_chat)
def cmd_export_followers(update: telegram.Update,
context: CallbackContext) -> None:
bot = context.bot
chat, _created = TelegramChat.get_or_create(
chat_id=update.message.chat.id,
tg_type=update.message.chat.type,
)
if not chat.is_authorized:
if not chat.twitter_request_token:
bot.reply(update,
"You have not authorized yet. Use /auth to do it")
else:
bot.reply(
update,
"You have not verified your authorization yet. Use /verify code to do it"
)
return
bot_auth = bot.tw.auth
api = chat.tw_api(bot_auth.consumer_key, bot_auth.consumer_secret)
me = api.me()
screen_names = [
f.screen_name for f in tweepy.Cursor(api.followers, count=200).items()
]
bot.reply(update,
"Use this to subscribe to all your Twitter subscriptions:")
bot.reply(update, "/sub {}".format(" ".join(screen_names)))
def cmd_set_timezone(update: telegram.Update,
context: CallbackContext) -> None:
args = context.args
bot = context.bot
chat, _created = TelegramChat.get_or_create(
chat_id=update.message.chat.id,
tg_type=update.message.chat.type,
)
if len(args) < 1:
bot.reply(update,
"No timezone specified. Find yours [here]({})!".format(
TIMEZONE_LIST_URL),
parse_mode=telegram.ParseMode.MARKDOWN)
return
tz_name = args[0]
try:
tz = timezone(tz_name)
chat.timezone_name = tz_name
chat.save()
tz_str = datetime.now(tz).strftime('%Z %z')
bot.reply(update, "Timezone is set to {}".format(tz_str))
except UnknownTimeZoneError:
bot.reply(update,
"Unknown timezone. Find yours [here]({})!".format(
TIMEZONE_LIST_URL),
parse_mode=telegram.ParseMode.MARKDOWN)
def cmd_all(update: telegram.Update, context: CallbackContext) -> None:
bot = context.bot
chat, _created = TelegramChat.get_or_create(
chat_id=update.message.chat.id,
tg_type=update.message.chat.type,
)
subscriptions = list(
Subscription.select().where(Subscription.tg_chat == chat))
if len(subscriptions) == 0:
return bot.reply(update,
'You have no subscriptions, so no tweets to show!')
text = ""
for sub in subscriptions:
if sub.last_tweet is None:
text += "\n{screen_name}: <no tweets yet>".format(
screen_name=escape_markdown(sub.tw_user.screen_name), )
else:
text += ("\n{screen_name}:\n{text} "
"[link](https://twitter.com/{screen_name}/status/{tw_id})"
).format(
text=markdown_twitter_usernames(
escape_markdown(sub.last_tweet.text)),
tw_id=sub.last_tweet.tw_id,
screen_name=escape_markdown(sub.tw_user.screen_name),
)
bot.reply(update,
text,
disable_web_page_preview=True,
parse_mode=telegram.ParseMode.MARKDOWN)
def cmd_verify(update: telegram.Update, context: CallbackContext) -> None:
args = context.args
bot = context.bot
chat, _created = TelegramChat.get_or_create(
chat_id=update.message.chat.id,
tg_type=update.message.chat.type,
)
if not chat.twitter_request_token:
bot.reply(update, "Use /auth command first")
return
if len(args) < 1:
bot.reply(update, "No verifier code specified")
return
verifier_code = args[0]
auth = OAuthHandler(bot.tw.auth.consumer_key, bot.tw.auth.consumer_secret)
auth.request_token = json.loads(chat.twitter_request_token)
try:
auth.get_access_token(verifier_code)
except TweepyException:
bot.reply(update, "Invalid verifier code. Use /auth again")
return
chat.twitter_token = auth.access_token
chat.twitter_secret = auth.access_token_secret
chat.save()
bot.reply(update, "Access token setup complete")
api = tweepy.API(auth)
settings = api.get_settings()
tz_name = settings.get("time_zone", {}).get("tzinfo_name")
def cmd_source(update: telegram.Update, context: CallbackContext) -> None:
bot = context.bot
chat, _created = TelegramChat.get_or_create(
chat_id=update.message.chat.id,
tg_type=update.message.chat.type,
)
bot.reply(
update, "This bot is Free Software under the LGPLv3. "
"You can get the code from here: "
"https://github.com/hoshinohikari/Twitter-To-Telegram-bot")
def cmd_get_auth_url(update: telegram.Update,
context: CallbackContext) -> None:
bot = context.bot
chat, _created = TelegramChat.get_or_create(
chat_id=update.message.chat.id,
tg_type=update.message.chat.type,
)
auth = OAuthHandler(bot.tw.auth.consumer_key, bot.tw.auth.consumer_secret)
auth_url = auth.get_authorization_url()
chat.twitter_request_token = json.dumps(auth.request_token)
chat.save()
msg = "go to [this url]({}) and send me your verifier code using /verify code"
bot.reply(update,
msg.format(auth_url),
parse_mode=telegram.ParseMode.MARKDOWN)
def cmd_sub_no_reply(update: telegram.Update,
context: CallbackContext) -> None:
args = context.args
bot = context.bot
chat, _created = TelegramChat.get_or_create(
chat_id=update.message.chat.id,
tg_type=update.message.chat.type,
)
if len(args) < 1:
bot.reply(update,
"Use /sub_no_reply username1 username2 username3 ...")
return
tw_usernames = args
not_found = []
already_subscribed = []
successfully_subscribed = []
for tw_username in tw_usernames:
tw_user = bot.get_tw_user(tw_username)
if tw_user is None:
not_found.append(tw_username)
continue
if Subscription.select().where(
Subscription.tw_user == tw_user,
Subscription.tg_chat == chat).count() == 1:
already_subscribed.append(tw_user.full_name)
continue
Subscription.create(tg_chat=chat, tw_user=tw_user, sub_kind=2)
successfully_subscribed.append(tw_user.full_name)
reply = ""
if len(not_found) != 0:
reply += "Sorry, I didn't find username{} {}\n\n".format(
"" if len(not_found) == 1 else "s", ", ".join(not_found))
if len(already_subscribed) != 0:
reply += "You're already subscribed to {}\n\n".format(
", ".join(already_subscribed))
if len(successfully_subscribed) != 0:
reply += "I've added your subscription to {}".format(
", ".join(successfully_subscribed))
bot.reply(update, reply)
def subfn(chat, match):
chat_model = TelegramChat.from_aiotg(chat)
args = match.group(1).strip()
with future_replies(chat) as add_reply:
try:
cls.validate(args)
except RuntimeError as e:
msg = ''.join(e.args)
add_reply("Whoops, that didn't work! {}".format(msg))
return
res = cls.bring_up(bot, args)
sub, sub_created = subscribe(chat_model, res)
add_reply(
"OK, subscribed!" if sub_created
else "You were already subscribed to that one!")
def cmd_subChan(bot, update, args, chat=None):
logging.getLogger("subChan").debug("subChan for {}".format(args))
if len(args) < 1:
bot.reply(
update,
"Use /sub username1 chatid , get it from https://api.telegram.org/bot<botid>/getUpdates after doing a post in the channel (public or private) with the robot disabled"
)
return
tw_username = args[0]
chatId = args[1]
newchat, _created = TelegramChat.get_or_create(chat_id=chatId,
tg_type="channel")
not_found = []
already_subscribed = []
successfully_subscribed = []
chat.chat_id = chatId
tw_user = bot.get_tw_user(tw_username)
if tw_user is None:
not_found.append(tw_username)
return
if Subscription.select().where(
Subscription.tw_user == tw_user,
Subscription.tg_chat == newchat).count() == 1:
already_subscribed.append(tw_user.full_name)
else:
Subscription.create(tg_chat=newchat, tw_user=tw_user)
successfully_subscribed.append(tw_user.full_name)
reply = ""
if len(not_found) is not 0:
reply += "Sorry, I didn't find username{} {}\n\n".format(
"" if len(not_found) is 1 else "s", ", ".join(not_found))
if len(already_subscribed) is not 0:
reply += "You're already subscribed to {}\n\n".format(
", ".join(already_subscribed))
if len(successfully_subscribed) is not 0:
reply += "I've added your subscription to {} with chat {} ".format(
", ".join(successfully_subscribed), chat.chat_id)
bot.reply(update, reply)
def cmd_wipe(update: telegram.Update, context: CallbackContext) -> None:
bot = context.bot
chat, _created = TelegramChat.get_or_create(
chat_id=update.message.chat.id,
tg_type=update.message.chat.type,
)
subscriptions = list(
Subscription.select().where(Subscription.tg_chat == chat))
subs = "You had no subscriptions."
if subscriptions:
subs = ''.join([
"For the record, you were subscribed to these users: ", ', '.join(
(s.tw_user.screen_name for s in subscriptions)), '.'
])
bot.reply(
update, "Okay, I'm forgetting about this chat. " + subs +
" Come back to me anytime you want. Goodbye!")
chat.delete_instance(recursive=True)
def cmd_export(update: telegram.Update, context: CallbackContext) -> None:
bot = context.bot
chat, _created = TelegramChat.get_or_create(
chat_id=update.message.chat.id,
tg_type=update.message.chat.type,
)
subscriptions = list(
Subscription.select().where(Subscription.tg_chat == chat))
if len(subscriptions) == 0:
return bot.reply(
update,
'You have no subscriptions yet! Add one with /sub username')
subs = ['']
for sub in subscriptions:
subs.append(sub.tw_user.screen_name)
subject = "Use this to subscribe to all subscribed Twitter users in another chat:\n\n"
bot.reply(update, subject + "/sub " + " ".join(subs))
def cmd_unsub(update: telegram.Update, context: CallbackContext) -> None:
args = context.args
bot = context.bot
chat, _created = TelegramChat.get_or_create(
chat_id=update.message.chat.id,
tg_type=update.message.chat.type,
)
if len(args) < 1:
bot.reply(update, "Use /unsub username1 username2 username3 ...")
return
tw_usernames = args
not_found = []
successfully_unsubscribed = []
for tw_username in tw_usernames:
tw_user = bot.get_tw_user(tw_username)
if tw_user is None or Subscription.select().where(
Subscription.tw_user == tw_user, Subscription.tg_chat
== chat).count() == 0:
not_found.append(tw_username)
continue
Subscription.delete().where(Subscription.tw_user == tw_user,
Subscription.tg_chat == chat).execute()
successfully_unsubscribed.append(tw_user.full_name)
reply = ""
if len(not_found) != 0:
reply += "I didn't find any subscription to {}\n\n".format(
", ".join(not_found))
if len(successfully_unsubscribed) != 0:
reply += "You are no longer subscribed to {}".format(
", ".join(successfully_unsubscribed))
bot.reply(update, reply)
def cmd_unChan(bot, update, args, chat=None):
if len(args) < 2:
bot.reply(update, "Use /unChan username1 channel")
return
tw_username = args[0]
chatID = args[1]
not_found = []
successfully_unsubscribed = []
tw_user = bot.get_tw_user(tw_username)
newchat = TelegramChat.select().where(TelegramChat.chat_id == chatID)
if newchat.count() == 0:
bot.reply("chat not found")
return
if tw_user is None or Subscription.select().where(
Subscription.tw_user == tw_user, Subscription.tg_chat
== newchat[0]).count() == 0:
not_found.append(tw_username)
bot.reply("chat and user not found")
return
Subscription.delete().where(Subscription.tw_user == tw_user,
Subscription.tg_chat == newchat[0]).execute()
successfully_unsubscribed.append(tw_user.full_name)
reply = ""
if len(not_found) is not 0:
reply += "I didn't find any subscription to {}\n\n".format(
", ".join(not_found))
if len(successfully_unsubscribed) is not 0:
reply += "You are no longer subscribed to {}".format(
", ".join(successfully_unsubscribed))
bot.reply(update, reply)
def cmd_list(update: telegram.Update, context: CallbackContext) -> None:
bot = context.bot
chat, _created = TelegramChat.get_or_create(
chat_id=update.message.chat.id,
tg_type=update.message.chat.type,
)
subscriptions = list(
Subscription.select().where(Subscription.tg_chat == chat))
if len(subscriptions) == 0:
return bot.reply(
update,
'You have no subscriptions yet! Add one with /sub username')
subs = ['']
for sub in subscriptions:
subs.append(sub.tw_user.full_name)
subject = "This group is" if chat.is_group else "You are"
bot.reply(
update, subject + " subscribed to the following Twitter users:\n" +
"\n - ".join(subs) +
"\n\nYou can remove any of them using /unsub username")
def run(self, bot):
self.logger.debug('Fetching tweets...')
tweet_rows = []
# fetch the tw users' tweets
tw_users = list((TwitterUser.select()
.join(Subscription)
.group_by(TwitterUser)
.order_by(TwitterUser.last_fetched)))
updated_tw_users = []
users_to_cleanup = []
for tw_user in tw_users:
try:
if tw_user.last_tweet_id == -1:
# get just the latest tweet
self.logger.debug(
'Fetching latest tweet by {}'.format(tw_user.screen_name))
tweets = bot.tw.user_timeline(
screen_name=tw_user.screen_name,
count=1,
tweet_mode='extended')
else:
# get the fresh tweets
self.logger.debug(
'Fetching new tweets from {}'.format(tw_user.screen_name))
tweets = bot.tw.user_timeline(
screen_name=tw_user.screen_name,
since_id=tw_user.last_tweet_id,
tweet_mode='extended')
updated_tw_users.append(tw_user)
except tweepy.error.TweepError as e:
sc = e.response.status_code
if sc == 429:
self.logger.debug('- Hit ratelimit, breaking.')
break
if sc == 401:
users_to_cleanup.append((tw_user, 'PROTECTED'))
self.logger.debug('- Protected tweets here. Cleaning up this user')
continue
if sc == 404:
users_to_cleanup.append((tw_user, 'NOTFOUND'))
self.logger.debug('- 404? Maybe screen name changed? Cleaning up this user')
continue
self.logger.debug(
'- Unknown exception, Status code {}'.format(sc))
continue
for tweet in tweets:
retweet = False
if 'retweeted_status' in tweet._json:
retweet = True
# NOTE: tweet.full_text doesn't work for retweets,
# see https://stackoverflow.com/a/48967803
# use current tweet by default
tweet_data = tweet
if retweet:
# use original tweet to text processing
tweet_data = tweet.retweeted_status
tw_text = tweet_data.full_text
self.logger.debug('- Got tweet: {}'.format(tw_text))
# Check if tweet contains media, else check if it contains a link to an image
extensions = ('.jpg', '.jpeg', '.png', '.gif')
pattern = '[(%s)]$' % ')('.join(extensions)
photo_url = ''
tweet_text = html.unescape(tw_text)
if 'media' in tweet_data.entities:
photo_url = tweet_data.entities['media'][0]['media_url_https']
else:
for url_entity in tweet_data.entities['urls']:
expanded_url = url_entity['expanded_url']
if re.search(pattern, expanded_url):
photo_url = expanded_url
break
if photo_url:
self.logger.debug('- - Found media URL in tweet: ' + photo_url)
for url_entity in tweet_data.entities['urls']:
expanded_url = url_entity['expanded_url']
parsed_url = urlparse(expanded_url)
indices = url_entity['indices']
display_url = tw_text[indices[0]:indices[1]]
replace_text = expanded_url
if parsed_url.netloc == 'twitter.com':
re_pattern = '/(?P<username>.+)/status/(?P<twit_id>[0-9]+)'
re_result = re.match(re_pattern, parsed_url.path)
if re_result != None:
try:
commented_tweet = bot.tw.get_status(re_result.group('twit_id'), tweet_mode='extended')
except:
pass
else:
# TODO: implement hack for retweets
# TODO: move text formatting into send_tweet, store only basic tweet bits
tweet_text = 'comment:\n' + tweet_text
replace_text = '\n\noriginal tweet:\n«{}»'.format(commented_tweet.full_text)
tweet_text = tweet_text.replace(display_url, replace_text)
tw_data = {
'tw_id': tweet.id,
'text': tweet_text,
'created_at': tweet.created_at,
'twitter_user': tw_user,
'photo_url': photo_url,
}
if retweet:
# store original screen name
tw_data['original_name'] = tweet_data.user.screen_name
try:
t = Tweet.get(Tweet.tw_id == tweet.id)
self.logger.warning('Got duplicated tw_id on this tweet:')
self.logger.warning(str(tw_data))
except Tweet.DoesNotExist:
tweet_rows.append(tw_data)
if len(tweet_rows) >= self.TWEET_BATCH_INSERT_COUNT:
Tweet.insert_many(tweet_rows).execute()
tweet_rows = []
TwitterUser.update(last_fetched=datetime.now()) \
.where(TwitterUser.id << [tw.id for tw in updated_tw_users]).execute()
if updated_tw_users and tweet_rows:
Tweet.insert_many(tweet_rows).execute()
# send last tweets to newcomers
subscriptions = list(Subscription.select()
.where(Subscription.last_tweet_id == -1))
for s in subscriptions:
self.logger.debug(
'Checking new subscription {} {}'.format(s.tg_chat.chat_id, s.tw_user.screen_name))
try:
tw = s.tw_user.tweets.select() \
.order_by(Tweet.tw_id.desc()) \
.first()
if tw is None:
s.last_tweet_id = 0
s.save()
self.logger.warning('Something fishy is going on here...')
else:
bot.send_tweet(s.tg_chat, tw)
# save the latest tweet sent on this subscription
s.last_tweet_id = tw.tw_id
s.save()
except IndexError:
self.logger.debug('- No tweets available yet on {}'.format(s.tw_user.screen_name))
# send the new tweets to existing subscribers
query = '''SELECT S.* FROM subscription S
INNER JOIN twitteruser TU
ON S.tw_user_id = TU.id
WHERE S.last_tweet_id <
(
SELECT tw_id
FROM tweet T
WHERE T.twitter_user_id = TU.id
ORDER BY T.tw_id DESC
LIMIT 1
)
'''
subscriptions = list(Subscription.raw(query))
for s in subscriptions:
# are there new tweets? send em all!
self.logger.debug(
'Checking subscription {} {}'.format(s.tg_chat.chat_id, s.tw_user.screen_name))
self.logger.debug('- Some fresh tweets here which was not sended yet!')
last_sended_tweet_id = s.last_tweet_id
for tw in (s.tw_user.tweets.select()
.where(Tweet.tw_id > s.last_tweet_id)
.order_by(Tweet.tw_id.asc())
):
if bot.send_tweet(s.tg_chat, tw):
last_sended_tweet_id = tw.tw_id
else:
break
# save the latest tweet sent on this subscription
s.last_tweet_id = last_sended_tweet_id
s.save()
continue
self.logger.debug('- No new tweets here.')
self.logger.debug('Starting tw_user cleanup')
if not users_to_cleanup:
self.logger.debug('- Nothing to cleanup')
else:
for tw_user, reason in users_to_cleanup:
self.logger.debug('- Cleaning up subs on user @{}, {}'.format(tw_user.screen_name, reason))
message = INFO_CLEANUP[reason].format(tw_user.screen_name)
subs = list(tw_user.subscriptions)
for s in subs:
chat = s.tg_chat
if chat.delete_soon:
self.logger.debug ('- - skipping because of delete_soon chatid={}'.format(chat_id))
continue
chat_id = chat.chat_id
self.logger.debug ('- - bye on chatid={}'.format(chat_id))
s.delete_instance()
try:
bot.sendMessage(chat_id=chat_id, text=message)
except TelegramError as e:
self.logger.info('Couldn\'t send unsubscription notice of {} to chat {}: {}'.format(
tw_user.screen_name, chat_id, e.message
))
delet_this = None
if e.message == 'Bad Request: group chat was migrated to a supergroup chat':
delet_this = True
if e.message == 'Unauthorized':
delet_this = True
if delet_this:
self.logger.info('Marking chat for deletion')
chat.delete_soon = True
chat.save()
self.logger.debug('- Cleaning up TwitterUser @{}'.format(tw_user.screen_name, reason))
tw_user.delete_instance()
self.logger.debug ('- User cleanup finished')
self.logger.debug('Cleaning up TelegramChats marked for deletion')
for chat in TelegramChat.select().where(TelegramChat.delete_soon == True):
chat.delete_instance(recursive=True)
self.logger.debug('Deleting chat {}'.format(chat.chat_id))
self.logger.debug('TelegramChats cleanup finished')
def get_chat(self, tg_chat):
db_chat, _created = TelegramChat.get_or_create(
chat_id=tg_chat.id,
tg_type=tg_chat.type,
)
return db_chat
def FetchAndSendTweetsJob(context_in: CallbackContext) -> None:
job = context_in.job
bot = context_in.bot
job.repeat = True
job.context = None
job.name = "FetchAndSendTweetsJob"
job._remove = Event()
job._enabled = Event()
job._enabled.set()
job.logger = logging.getLogger(job.name)
job.logger.debug("Fetching tweets...")
tweet_rows = []
# fetch the tw users' tweets
tw_users = list((
TwitterUser.select().join(Subscription).group_by(TwitterUser).order_by(
TwitterUser.last_fetched)))
updated_tw_users = []
users_to_cleanup = []
for tw_user in tw_users:
try:
if tw_user.last_tweet_id == 0:
# get just the latest tweet
job.logger.debug("Fetching latest tweet by {}".format(
tw_user.screen_name))
tweets = bot.tw.user_timeline(screen_name=tw_user.screen_name,
count=1,
tweet_mode='extended')
else:
# get the fresh tweets
job.logger.debug("Fetching new tweets from {}".format(
tw_user.screen_name))
tweets = bot.tw.user_timeline(screen_name=tw_user.screen_name,
since_id=tw_user.last_tweet_id,
tweet_mode='extended')
updated_tw_users.append(tw_user)
except tweepy.errors.TweepyException as e:
sc = e.response.status_code
if sc == 429:
job.logger.debug("- Hit ratelimit, breaking.")
break
if sc == 401:
users_to_cleanup.append((tw_user, 'PROTECTED'))
job.logger.debug(
"- Protected tweets here. Cleaning up this user")
continue
if sc == 404:
users_to_cleanup.append((tw_user, 'NOTFOUND'))
job.logger.debug(
"- 404? Maybe screen name changed? Cleaning up this user")
continue
job.logger.debug("- Unknown exception, Status code {}".format(sc))
continue
for tweet in tweets:
job.logger.debug("- Got tweet: {}".format(tweet.full_text))
# Check if tweet contains media, else check if it contains a link to an image
extensions = ('.jpg', '.jpeg', '.png', '.gif')
pattern = '[(%s)]$' % ')('.join(extensions)
photo_url = []
video_url = ''
tweet_text = html.unescape(tweet.full_text)
if 'media' in tweet.entities:
for imgs in tweet.extended_entities['media']:
photo_url.append(imgs['media_url_https'])
try:
if 'video_info' in tweet.extended_entities['media'][0]:
# file = open("video_url.txt", "a")
# file.write('\n')
# file.write(str(tweet.extended_entities['media']))
# file.close()
max_bit = 0
for video_info in tweet.extended_entities['media'][0][
'video_info']['variants']:
if 'bitrate' in video_info:
if (video_info['bitrate'] > max_bit):
video_url = video_info['url']
max_bit = video_info['bitrate']
except:
job.logger.warning(
"{} Finding video failed, video url is in the video_url.txt..."
.format(tweet.id))
else:
for url_entity in tweet.entities['urls']:
expanded_url = url_entity['expanded_url']
if re.search(pattern, expanded_url):
photo_url.append(expanded_url)
break
if len(photo_url) != 0:
job.logger.debug("- - Found media URL in tweet: " +
photo_url[0])
for url_entity in tweet.entities['urls']:
expanded_url = url_entity['expanded_url']
indices = url_entity['indices']
display_url = tweet.full_text[indices[0]:indices[1]]
tweet_text = tweet_text.replace(display_url, expanded_url)
tw_data = {
'tw_id': tweet.id,
'text': tweet_text,
'created_at': tweet.created_at,
'twitter_user': tw_user,
'photo_url': photo_url,
'video_url': video_url,
}
try:
t = Tweet.get(Tweet.tw_id == tweet.id)
job.logger.warning("Got duplicated tw_id on this tweet:")
job.logger.warning(str(tw_data))
except Tweet.DoesNotExist:
tweet_rows.append(tw_data)
if len(tweet_rows) >= 100:
Tweet.insert_many(tweet_rows).execute()
tweet_rows = []
TwitterUser.update(last_fetched=datetime.now()) \
.where(TwitterUser.id << [tw.id for tw in updated_tw_users]).execute()
if not updated_tw_users:
return
if tweet_rows:
Tweet.insert_many(tweet_rows).execute()
# send the new tweets to subscribers
subscriptions = list(
Subscription.select().where(Subscription.tw_user << updated_tw_users))
for s in subscriptions:
# are there new tweets? send em all!
job.logger.debug("Checking subscription {} {}".format(
s.tg_chat.chat_id, s.tw_user.screen_name))
if s.last_tweet_id == 0: # didn't receive any tweet yet
try:
tw = s.tw_user.tweets.select() \
.order_by(Tweet.tw_id.desc()) \
.first()
if tw is None:
job.logger.warning("Something fishy is going on here...")
else:
bot.send_tweet(s.tg_chat, tw, s.sub_kind)
# save the latest tweet sent on this subscription
s.last_tweet_id = tw.tw_id
s.save()
except IndexError:
job.logger.debug("- No tweets available yet on {}".format(
s.tw_user.screen_name))
continue
if s.tw_user.last_tweet_id > s.last_tweet_id:
job.logger.debug("- Some fresh tweets here!")
for tw in (s.tw_user.tweets.select().where(
Tweet.tw_id > s.last_tweet_id).order_by(
Tweet.tw_id.asc())):
bot.send_tweet(s.tg_chat, tw, s.sub_kind)
# save the latest tweet sent on this subscription
s.last_tweet_id = s.tw_user.last_tweet_id
s.save()
continue
job.logger.debug("- No new tweets here.")
job.logger.debug("Starting tw_user cleanup")
if not users_to_cleanup:
job.logger.debug("- Nothing to cleanup")
else:
for tw_user, reason in users_to_cleanup:
job.logger.debug("- Cleaning up subs on user @{}, {}".format(
tw_user.screen_name, reason))
message = INFO_CLEANUP[reason].format(tw_user.screen_name)
subs = list(tw_user.subscriptions)
for s in subs:
chat = s.tg_chat
if chat.delete_soon:
job.logger.debug(
"- - skipping because of delete_soon chatid={}".format(
chat_id))
continue
chat_id = chat.chat_id
job.logger.debug("- - bye on chatid={}".format(chat_id))
s.delete_instance()
try:
bot.sendMessage(chat_id=chat_id, text=message)
except TelegramError as e:
job.logger.info(
"Couldn't send unsubscription notice of {} to chat {}: {}"
.format(tw_user.screen_name, chat_id, e.message))
delet_this = None
if e.message == 'Bad Request: group chat was migrated to a supergroup chat':
delet_this = True
if e.message == "Unauthorized":
delet_this = True
if delet_this:
job.logger.info("Marking chat for deletion")
chat.delete_soon = True
chat.save()
job.logger.debug("- Cleaning up TwitterUser @{} {}".format(
tw_user.screen_name, reason))
tw_user.delete_instance()
job.logger.debug("- Cleanup finished")
job.logger.debug("Cleaning up TelegramChats marked for deletion")
for chat in TelegramChat.select().where(TelegramChat.delete_soon == True):
chat.delete_instance(recursive=True)
job.logger.debug("Deleting chat {}".format(chat.chat_id))
def get_chat(self, tg_chat):
db_chat, _created = TelegramChat.get_or_create(
chat_id=tg_chat.id,
tg_type=tg_chat.type,
)
return db_chat
def update(self):
self.logger.debug("Cleaning up TelegramChats marked for deletion")
for chat in TelegramChat.select().where(
TelegramChat.delete_soon == True):
chat.delete_instance(recursive=True)
self.logger.debug("Deleting chat {}".format(chat.chat_id))
def run(self, bot):
self.logger.debug("Fetching tweets...")
tweet_rows = []
# fetch the tg users' home timelines
tg_chats = list(TelegramChat.select().where(TelegramChat.twitter_secret != None))
for tg_chat in tg_chats:
bot_auth = bot.tw.auth
tw_api = tg_chat.tw_api(bot_auth.consumer_key, bot_auth.consumer_secret)
try:
if tg_chat.last_tweet_id == 0:
# get just the 5 latest tweets
self.logger.debug(
"Fetching 5 latest tweets for {}".format(tg_chat.chat_id))
tweets = tw_api.home_timeline(count=5, tweet_mode='extended')
else:
# get the fresh tweets
self.logger.debug(
"Fetching new tweets from {}".format(tg_chat.last_tweet_id))
tweets = tw_api.home_timeline(since_id=tg_chat.last_tweet_id, tweet_mode='extended')
except tweepy.error.TweepError as e:
sc = e.response.status_code
if sc == 429:
self.logger.debug("- Hit ratelimit, breaking.")
break
self.logger.debug(
"- Unknown exception, Status code {}".format(sc))
continue
sorted_tweets = sorted(tweets, key=lambda tweet: tweet.id)
for tweet in sorted_tweets:
self.logger.debug("- Got tweet #{}: {}".format(tweet.id, tweet.full_text))
# Check if tweet contains media, else check if it contains a link to an image
extensions = ('.jpg', '.jpeg', '.png', '.gif')
pattern = '[(%s)]$' % ')('.join(extensions)
video_url = ''
photo_url = ''
link_url = ''
tweet_text = html.unescape(tweet.full_text)
if hasattr(tweet, 'retweeted_status'):
tweet_text = u'\U0001F501' + ' @' + tweet.retweeted_status.user.screen_name + ': ' + html.unescape(tweet.retweeted_status.full_text)
if hasattr(tweet, 'quoted_status'):
tweet_text += "\n" + u'\U0001F501' + ' @' + tweet.quoted_status['user']['screen_name'] + ': ' + html.unescape(tweet.quoted_status['full_text'])
tweet.entities['urls'] = []
if hasattr(tweet, 'extended_entities') and 'media' in tweet.extended_entities:
first_media = tweet.extended_entities['media'][0]
tweet_text = tweet_text.replace(first_media['url'], '')
if 'video_info' in first_media:
video_urls = first_media['video_info']['variants']
video_url = max([video for video in video_urls if ('bitrate') in d ],key=lambda x:x['bitrate'])['url']
else:
photo_url = first_media['media_url_https']
elif tweet.entities['urls']:
link_url = tweet.entities['urls'][0]['expanded_url']
if video_url:
self.logger.debug("- - Found video URL in tweet: " + video_url)
if photo_url:
self.logger.debug("- - Found photo URL in tweet: " + photo_url)
for url_entity in tweet.entities['urls']:
expanded_url = url_entity['expanded_url']
indices = url_entity['indices']
display_url = tweet.full_text[indices[0]:indices[1]]
tweet_text = tweet_text.replace(display_url, expanded_url)
t = Tweet(
tw_id=tweet.id,
text=tweet_text,
created_at=tweet.created_at,
# twitter_user=tweet.id,
twitter_user_name=tweet.author.name,
twitter_user_screen_name=tweet.author.screen_name,
photo_url=photo_url,
video_url=video_url,
link_url=link_url
)
# save the latest tweet sent to chat
self.logger.debug("- Setting id: {}".format(tweet.id))
tg_chat.last_tweet_id = tweet.id
bot.send_tweet(tg_chat, t)
tg_chat.save()
self.logger.debug("Cleaning up TelegramChats marked for deletion")
for chat in TelegramChat.select().where(TelegramChat.delete_soon == True):
chat.delete_instance(recursive=True)
self.logger.debug("Deleting chat {}".format(chat.chat_id))
def run(self, bot):
self.logger.debug("Fetching tweets...")
tweet_rows = []
tw_users = list((TwitterUser.select()
.join(Subscription)
.group_by(TwitterUser)
.order_by(TwitterUser.last_fetched)))
updated_tw_users = []
users_to_cleanup = []
for tw_user in tw_users:
try:
if tw_user.last_tweet_id == 0:
self.logger.debug("Fetching latest tweet by {}".format(tw_user.screen_name))
tweets = bot.tw.user_timeline(screen_name=tw_user.screen_name,
count=1,
tweet_mode='extended',
include_rts=False,
exclude_replies=True)
else:
self.logger.debug("Fetching new tweets from {}".format(tw_user.screen_name))
tweets = bot.tw.user_timeline(screen_name=tw_user.screen_name,
since_id=tw_user.last_tweet_id,
tweet_mode='extended',
include_rts=False,
exclude_replies=True)
updated_tw_users.append(tw_user)
except tweepy.error.TweepError as e:
sc = e.response.status_code
if sc == 429:
self.logger.debug("- Hit rate limit, breaking.")
break
if sc == 401:
users_to_cleanup.append((tw_user, 'PROTECTED'))
self.logger.debug("- Protected tweets here. Cleaning up this user")
continue
if sc == 404:
users_to_cleanup.append((tw_user, 'NOTFOUND'))
self.logger.debug("- 404? Maybe screen name changed? Cleaning up this user")
continue
self.logger.debug("- Unknown exception, Status code {}".format(sc))
continue
for tweet in tweets:
self.logger.debug("- Got tweet: {}".format(tweet.full_text))
# Check if tweet contains media, else check if it contains a link to an image
extensions = ('.jpg', '.jpeg', '.png', '.gif')
pattern = '[(%subscription)]$' % ')('.join(extensions)
photo_url = ''
tweet_text = html.unescape(tweet.full_text)
if 'media' in tweet.entities:
photo_url = tweet.entities['media'][0]['media_url_https']
else:
for url_entity in tweet.entities['urls']:
expanded_url = url_entity['expanded_url']
if re.search(pattern, expanded_url):
photo_url = expanded_url
break
if photo_url:
self.logger.debug("- - Found media URL in tweet: " + photo_url)
for url_entity in tweet.entities['urls']:
expanded_url = url_entity['expanded_url']
indices = url_entity['indices']
display_url = tweet.full_text[indices[0]:indices[1]]
tweet_text = tweet_text.replace(display_url, expanded_url)
tw_data = {
'tw_id': tweet.id,
'text': tweet_text,
'created_at': tweet.created_at,
'twitter_user': tw_user,
'photo_url': photo_url,
}
try:
t = Tweet.get(Tweet.tw_id == tweet.id)
self.logger.warning("Got duplicated tw_id on this tweet:")
self.logger.warning(str(tw_data))
except Tweet.DoesNotExist:
tweet_rows.append(tw_data)
if len(tweet_rows) >= self.TWEET_BATCH_INSERT_COUNT:
Tweet.insert_many(tweet_rows).execute()
tweet_rows = []
TwitterUser \
.update(last_fetched=datetime.now()) \
.where(TwitterUser.id << [tw.id for tw in updated_tw_users]) \
.execute()
if not updated_tw_users:
return
if tweet_rows:
Tweet.insert_many(tweet_rows).execute()
# send new tweets to subscribers
subscriptions = list(Subscription.select().where(Subscription.tw_user << updated_tw_users))
for subscription in subscriptions:
# send all new tweets
self.logger.debug("Checking subscription {} {}".format(subscription.tg_chat.chat_id, subscription.tw_user.screen_name))
if subscription.last_tweet_id == 0: # didn't receive any tweet yet
try:
tw = subscription.tw_user.tweets.select() \
.order_by(Tweet.tw_id.desc()) \
.first()
if tw is None:
self.logger.warning("Something fishy is going on here...")
else:
bot.send_tweet(subscription.tg_chat, tw)
# save the latest tweet sent on this subscription
subscription.last_tweet_id = tw.tw_id
subscription.save()
except IndexError:
self.logger.debug("- No tweets available yet on {}".format(subscription.tw_user.screen_name))
continue
if subscription.tw_user.last_tweet_id > subscription.last_tweet_id:
self.logger.debug("- Some fresh tweets here!")
for tw in (subscription.tw_user.tweets.select().where(Tweet.tw_id > subscription.last_tweet_id).order_by(Tweet.tw_id.asc())):
bot.send_tweet(subscription.tg_chat, tw)
# save the latest tweet sent on this subscription
subscription.last_tweet_id = subscription.tw_user.last_tweet_id
subscription.save()
continue
self.logger.debug("- No new tweets here.")
self.logger.debug("Starting tw_user cleanup")
if not users_to_cleanup:
self.logger.debug("- Nothing to cleanup")
else:
for tw_user, reason in users_to_cleanup:
self.logger.debug("- Cleaning up subs on user @{}, {}".format(tw_user.screen_name, reason))
message = INFO_CLEANUP[reason].format(tw_user.screen_name)
subs = list(tw_user.subscriptions)
for subscription in subs:
chat = subscription.tg_chat
if chat.delete_soon:
self.logger.debug("- - skipping because of delete_soon chat id={}".format(chat_id))
continue
chat_id = chat.chat_id
self.logger.debug("- - bye on chat id={}".format(chat_id))
subscription.delete_instance()
try:
bot.sendMessage(chat_id=chat_id, text=message)
except TelegramError as e:
self.logger.info("Couldn't send unsubscription notice of {} to chat {}: {}"
.format(tw_user.screen_name, chat_id, e.message))
delete_this = None
if e.message == 'Bad Request: group chat was migrated to a supergroup chat':
delete_this = True
if e.message == "Unauthorized":
delete_this = True
if delete_this:
self.logger.info("Marking chat for deletion")
chat.delete_soon = True
chat.save()
self.logger.debug("- Cleaning up TwitterUser @{}".format(tw_user.screen_name, reason))
tw_user.delete_instance()
self.logger.debug("- Cleanup finished")
self.logger.debug("Cleaning up TelegramChats marked for deletion")
for chat in TelegramChat.select().where(TelegramChat.delete_soon == True):
chat.delete_instance(recursive=True)
self.logger.debug("Deleting chat {}".format(chat.chat_id))
def run(self, bot):
self.logger.debug("Fetching tweets...")
tweet_rows = []
# fetch the tw users' tweets
tw_users = list((TwitterUser.select()
.join(Subscription)
.group_by(TwitterUser)
.order_by(TwitterUser.last_fetched)))
updated_tw_users = []
users_to_cleanup = []
for tw_user in tw_users:
try:
if tw_user.last_tweet_id == 0:
# get just the latest tweet
self.logger.debug(
"Fetching latest tweet by {}".format(tw_user.screen_name))
tweets = bot.tw.user_timeline(
screen_name=tw_user.screen_name,
count=1)
else:
# get the fresh tweets
self.logger.debug(
"Fetching new tweets from {}".format(tw_user.screen_name))
tweets = bot.tw.user_timeline(
screen_name=tw_user.screen_name,
since_id=tw_user.last_tweet_id)
updated_tw_users.append(tw_user)
except tweepy.error.TweepError as e:
sc = e.response.status_code
if sc == 429:
self.logger.debug("- Hit ratelimit, breaking.")
break
if sc == 401:
users_to_cleanup.append((tw_user, 'PROTECTED'))
self.logger.debug("- Protected tweets here. Cleaning up this user")
continue
if sc == 404:
users_to_cleanup.append((tw_user, 'NOTFOUND'))
self.logger.debug("- 404? Maybe screen name changed? Cleaning up this user")
continue
self.logger.debug(
"- Unknown exception, Status code {}".format(sc))
continue
for tweet in tweets:
self.logger.debug("- Got tweet: {}".format(tweet.text))
# Check if tweet contains media, else check if it contains a link to an image
extensions = ('.jpg', '.jpeg', '.png', '.gif')
pattern = '[(%s)]$' % ')('.join(extensions)
photo_url = ''
tweet_text = html.unescape(tweet.text)
if 'media' in tweet.entities:
photo_url = tweet.entities['media'][0]['media_url_https']
else:
for url_entity in tweet.entities['urls']:
expanded_url = url_entity['expanded_url']
if re.search(pattern, expanded_url):
photo_url = expanded_url
break
if photo_url:
self.logger.debug("- - Found media URL in tweet: " + photo_url)
for url_entity in tweet.entities['urls']:
expanded_url = url_entity['expanded_url']
indices = url_entity['indices']
display_url = tweet.text[indices[0]:indices[1]]
tweet_text = tweet_text.replace(display_url, expanded_url)
tw_data = {
'tw_id': tweet.id,
'text': tweet_text,
'created_at': tweet.created_at,
'twitter_user': tw_user,
'photo_url': photo_url,
}
try:
t = Tweet.get(Tweet.tw_id == tweet.id)
self.logger.warning("Got duplicated tw_id on this tweet:")
self.logger.warning(str(tw_data))
except Tweet.DoesNotExist:
tweet_rows.append(tw_data)
if len(tweet_rows) >= self.TWEET_BATCH_INSERT_COUNT:
Tweet.insert_many(tweet_rows).execute()
tweet_rows = []
TwitterUser.update(last_fetched=datetime.now()) \
.where(TwitterUser.id << [tw.id for tw in updated_tw_users]).execute()
if not updated_tw_users:
return
if tweet_rows:
Tweet.insert_many(tweet_rows).execute()
# send the new tweets to subscribers
subscriptions = list(Subscription.select()
.where(Subscription.tw_user << updated_tw_users))
for s in subscriptions:
# are there new tweets? send em all!
self.logger.debug(
"Checking subscription {} {}".format(s.tg_chat.chat_id, s.tw_user.screen_name))
if s.last_tweet_id == 0: # didn't receive any tweet yet
try:
tw = s.tw_user.tweets.select() \
.order_by(Tweet.tw_id.desc()) \
.first()
if tw is None:
self.logger.warning("Something fishy is going on here...")
else:
bot.send_tweet(s.tg_chat, tw)
# save the latest tweet sent on this subscription
s.last_tweet_id = tw.tw_id
s.save()
except IndexError:
self.logger.debug("- No tweets available yet on {}".format(s.tw_user.screen_name))
continue
if s.tw_user.last_tweet_id > s.last_tweet_id:
self.logger.debug("- Some fresh tweets here!")
for tw in (s.tw_user.tweets.select()
.where(Tweet.tw_id > s.last_tweet_id)
.order_by(Tweet.tw_id.asc())
):
bot.send_tweet(s.tg_chat, tw)
# save the latest tweet sent on this subscription
s.last_tweet_id = s.tw_user.last_tweet_id
s.save()
continue
self.logger.debug("- No new tweets here.")
self.logger.debug("Starting tw_user cleanup")
if not users_to_cleanup:
self.logger.debug("- Nothing to cleanup")
else:
for tw_user, reason in users_to_cleanup:
self.logger.debug("- Cleaning up subs on user @{}, {}".format(tw_user.screen_name, reason))
message = INFO_CLEANUP[reason].format(tw_user.screen_name)
subs = list(tw_user.subscriptions)
for s in subs:
chat = s.tg_chat
if chat.delete_soon:
self.logger.debug ("- - skipping because of delete_soon chatid={}".format(chat_id))
continue
chat_id = chat.chat_id
self.logger.debug ("- - bye on chatid={}".format(chat_id))
s.delete_instance()
try:
bot.sendMessage(chat_id=chat_id, text=message)
except TelegramError as e:
self.logger.info("Couldn't send unsubscription notice of {} to chat {}: {}".format(
tw_user.screen_name, chat_id, e.message
))
delet_this = None
if e.message == 'Bad Request: group chat was migrated to a supergroup chat':
delet_this = True
if e.message == "Unauthorized":
delet_this = True
if delet_this:
self.logger.info("Marking chat for deletion")
chat.delete_soon = True
chat.save()
self.logger.debug("- Cleaning up TwitterUser @{}".format(tw_user.screen_name, reason))
tw_user.delete_instance()
self.logger.debug ("- Cleanup finished")
self.logger.debug("Cleaning up TelegramChats marked for deletion")
for chat in TelegramChat.select().where(TelegramChat.delete_soon == True):
chat.delete_instance(recursive=True)
self.logger.debug("Deleting chat {}".format(chat.chat_id))
