def update_twitter_link_length(self):
if time.time() - self.last_url_len_update > 60 * 60 * 24:
self.twitter_api._config = None
self.url_length = max(self.twitter_api.GetShortUrlLength(False),
self.twitter_api.GetShortUrlLength(True)) + 1
self.last_url_len_update = time.time()
lgt(f'Updated expected short URL length - it is now {self.url_length} characters.')
def init_process(self):
try:
self.since_toot_id = self.mastodon_api.account_statuses(self.ma_account_id)[0]["id"]
lgt(f'Tweeting any toot after toot {self.since_toot_id}')
except IndexError:
lgt('Tweeting any toot (user timeline is empty right now)')
self.update_twitter_link_length()
def run(self):
self.init_process()
lgt('Listening for tweets…')
for tweet in self.twitter_api.GetUserStream():
if 'text' not in tweet and 'full_text' not in tweet:
continue
# Avoids a race condition.
# We wait a little bit so toots sent to Twitter
# can be marked as such before this run, avoiding
# bouncing tweets/toots.
time.sleep(config.STATUS_PROCESS_DELAY)
tweet_id = tweet['id']
if self.is_tweet_sent_by_us(tweet_id):
continue
if tweet['user']['id_str'] != str(self.tw_account_id):
continue
is_retweet = False
content = MastodonPublisher._get_tweet_full_text(tweet)
if 'retweeted_status' in tweet:
rt = tweet['retweeted_status']
rt_content = MastodonPublisher._get_tweet_full_text(rt)
content = f'\U0001f501 RT @{rt["user"]["screen_name"]}\n\n' \
f'{rt_content}\n\n' \
f'https://twitter.com/{rt["user"]["screen_name"]}/status/{rt["id_str"]}'
tweet = rt
is_retweet = True
reply_to = None
with lock:
if 'in_reply_to_user_id' in tweet and 'in_reply_to_status_id' in tweet and tweet['in_reply_to_user_id']:
# If it's a reply, we keep the tweet if:
# 1. it's a reply from us (in a thread);
# 2. it's a reply from a previously transmitted tweet, so we don't sync
# if someone replies to someone in two or more tweets (because in this
# case the 2nd tweet and the ones after are replying to us);
# 3. it's a reply from another one but we retweeted it.
# If it's not a tweet in reply to us
if ((tweet['in_reply_to_user_id'] != self.tw_account_id
# or if it's a reply to us but not in our threads association
or tweet['in_reply_to_status_id'] not in self.status_associations['t2m'])
# or if it's a tweet from us but not a retweet
and not is_retweet):
# ... in all these cases, we don't want to transfer the tweet.
lgt(f'Skipping tweet {tweet_id} - it\'s a reply.')
continue
# A tweet can be a reply without previous tweet if we directly mentioned someone
# (starting the tweet with the mention).
if tweet['in_reply_to_status_id'] is not None:
reply_to = self.status_associations['t2m'].get(tweet['in_reply_to_status_id'])
media_attachments = (tweet['media'] if 'media' in tweet
else tweet['entities']['media'] if 'entities' in tweet and 'media' in tweet['entities']
else tweet['extended_tweet']['entities']['media']
if 'extended_tweet' in tweet and 'entities' in tweet['extended_tweet']
and 'media' in tweet['extended_tweet']['entities']
else [])
urls = (tweet['urls'] if 'urls' in tweet
else tweet['entities']['urls'] if 'entities' in tweet and 'urls' in tweet['entities']
else tweet['extended_tweet']['entities']['urls']
if 'extended_tweet' in tweet and 'entities' in tweet['extended_tweet']
and 'urls' in tweet['extended_tweet']['entities']
else [])
sensitive = tweet['possibly_sensitive'] if 'possibly_sensitive' in tweet else False
content_toot = html.unescape(content)
mentions = re.findall(r'@[a-zA-Z0-9_]*', content_toot)
cws = config.TWEET_CW_REGEXP.findall(content) if config.TWEET_CW_REGEXP else []
warning = None
media_ids = []
if mentions:
for mention in mentions:
# Replace all mentions for an equivalent to clearly signal their origin on Twitter
content_toot = re.sub(mention, mention + '@twitter.com', content_toot)
if urls:
for url in urls:
# Un-shorten URLs
content_toot = re.sub(url['url'], url['expanded_url'], content_toot)
if cws:
warning = (config.TWEET_CW_SEPARATOR.join([cw.strip() for cw in cws]) if config.TWEET_CW_ALLOW_MULTI
else cws[0].strip())
content_toot = config.TWEET_CW_REGEXP.sub('', content_toot, count=(0 if config.TWEET_CW_ALLOW_MULTI
else 1)).strip()
if media_attachments:
for attachment in media_attachments:
# Remove the t.co link to the media
content_toot = re.sub(attachment['url'], '', content_toot)
attachment_url = (attachment['media_url_https'] if 'media_url_https' in attachment
else attachment['media_url'])
media_ids.append(self.transfer_media(
media_url=attachment_url,
to='mastodon'
))
# Now that the toot is ready, we send it.
try:
retry_counter = 0
post_success = False
lgt(f'Sending toot "{content_toot.strip()}"…')
while not post_success:
try:
if len(media_ids) == 0:
try:
post = self.mastodon_api.status_post(
content_toot,
visibility=config.TOOT_VISIBILITY,
spoiler_text=warning,
in_reply_to_id=reply_to
)
self.mark_toot_sent(post['id'])
except MastodonAPIError:
# If the toot we are replying to has been deleted while we were processing it
post = self.mastodon_api.status_post(
content_toot,
visibility=config.TOOT_VISIBILITY,
spoiler_text=warning
)
self.mark_toot_sent(post['id'])
since_toot_id = post['id']
post_success = True
else:
try:
post = self.mastodon_api.status_post(
content_toot,
media_ids=media_ids,
visibility=config.TOOT_VISIBILITY,
sensitive=sensitive,
spoiler_text=warning,
in_reply_to_id=reply_to
)
self.mark_toot_sent(post['id'])
except MastodonAPIError:
# If the toot we are replying to has been deleted (same as before)
post = self.mastodon_api.status_post(
content_toot,
media_ids=media_ids,
visibility=config.TOOT_VISIBILITY,
sensitive=sensitive,
spoiler_text=warning
)
self.mark_toot_sent(post['id'])
since_toot_id = post['id']
post_success = True
except MastodonError:
if retry_counter < config.TWITTER_RETRIES:
lgt('We were unable to send the toot. '
f'Retrying… ({retry_counter+1}/{config.TWITTER_RETRIES})')
retry_counter += 1
time.sleep(config.TWITTER_RETRY_DELAY)
else:
raise
lgt('Toot sent successfully.')
with lock:
self.associate_status(since_toot_id, tweet_id)
self.save_status_associations()
except MastodonError:
lgt(f'Encountered error after {config.TWITTER_RETRIES} retries. Not retrying.')
# Broad exception to avoid thread interruption in case of network problems or anything else.
except Exception as e:
lgt('Unhandled exception happened - giving up on this toot.')
lgt(e)
def init_process(self):
try:
self.since_tweet_id = self.twitter_api.GetUserTimeline()[0].id
lgt('Tooting any tweet after tweet {}'.format(self.since_tweet_id))
except IndexError:
lgt('Tooting any tweet (user timeline is empty right now)')
def on_update(self, toot):
# We only transfer our own toots, but the streaming endpoint receives the whole
# timeline.
if not self.publisher.is_from_us(toot['account']):
return
# Avoids a race condition.
# We wait a little bit so tweets sent to Mastodon
# can be marked as such before this run, avoiding
# bouncing tweets/toots.
time.sleep(config.STATUS_PROCESS_DELAY)
toot_id = toot["id"]
if self.publisher.is_toot_sent_by_us(toot_id):
return
if toot['visibility'] not in config.TOOT_VISIBILITY_REQUIRED_TO_TRANSFER:
lgt(f'Skipping toot {toot["id"]} - invalid visibility ({toot["visibility"]})'
)
return
content = toot["content"]
media_attachments = toot["media_attachments"]
if toot['reblogged'] and 'reblog' in toot:
reblog = toot['reblog']
reblog_name = f'@{reblog["account"]["username"]}@{urlparse(reblog["account"]["url"]).netloc}'
content = f'\U0001f501 RT {reblog_name}\n' \
f'{reblog["content"]}\n\n' \
f'{reblog["url"]}'
media_attachments = reblog["media_attachments"]
toot = reblog
# We trust mastodon to return valid HTML
content_clean = re.sub(r'<a [^>]*href="([^"]+)">[^<]*</a>',
'\g<1>', content)
# We replace html br with new lines
content_clean = "\n".join(
re.compile(r'<br ?/?>',
re.IGNORECASE).split(content_clean))
# We must also replace new paragraphs with double line skips
content_clean = "\n\n".join(
re.compile(r'</p><p>', re.IGNORECASE).split(content_clean))
# Then we can delete the other html contents and unescape the string
content_clean = html.unescape(
str(re.compile(r'<.*?>').sub("", content_clean).strip()))
# Trim out media URLs
content_clean = re.sub(self.publisher.MEDIA_REGEXP, "",
content_clean)
# Don't cross-post replies
if len(content_clean) != 0 and content_clean[0] == '@':
lgt('Skipping toot "' + content_clean + '" - is a reply.')
return
if config.TWEET_CW_PREFIX and toot['spoiler_text']:
content_clean = config.TWEET_CW_PREFIX.format(
toot['spoiler_text']) + content_clean
content_parts = split_status(status=content_clean,
max_length=280,
split=config.SPLIT_ON_TWITTER,
url=toot['uri'])
# Tweet all the parts. On error, give up and go on with the next toot.
try:
reply_to = None
# We check if this toot is a reply to a previously sent toot.
# If so, the first corresponding tweet will be a reply to
# the stored tweet.
# Unlike in the Mastodon API calls, we don't have to handle the
# case where the tweet was deleted, as twitter will ignore
# the in_reply_to_status_id option if the given tweet
# does not exists.
if toot['in_reply_to_id'] in self.publisher.status_associations[
'm2t']:
reply_to = self.publisher.status_associations['m2t'][
toot['in_reply_to_id']]
for i in range(len(content_parts)):
media_ids = []
content_tweet = content_parts[i]
# Last content part: Upload media, no -- at the end
if i == len(content_parts) - 1:
for attachment in media_attachments:
media_ids.append(
self.publisher.transfer_media(
media_url=attachment["url"],
to='twitter'))
content_tweet = content_parts[i]
# Some final cleaning
content_tweet = content_tweet.strip()
# Retry three times before giving up
retry_counter = 0
post_success = False
lgt(f'Sending tweet "{content_tweet}"…')
while not post_success:
try:
if len(media_ids) == 0:
reply_to = self.publisher.twitter_api.PostUpdate(
content_tweet,
in_reply_to_status_id=reply_to).id
self.publisher.mark_tweet_sent(reply_to)
since_tweet_id = reply_to
post_success = True
else:
reply_to = self.publisher.twitter_api.PostUpdate(
content_tweet,
media=media_ids,
in_reply_to_status_id=reply_to).id
self.publisher.mark_tweet_sent(reply_to)
since_tweet_id = reply_to
post_success = True
except TwitterError:
if retry_counter < config.MASTODON_RETRIES:
retry_counter += 1
time.sleep(config.MASTODON_RETRY_DELAY)
else:
raise
lgt('Tweet sent successfully.')
# Only the last tweet is linked to the toot, see comment
# above the status_associations declaration
if i == len(content_parts) - 1:
with lock:
self.publisher.associate_status(
toot_id, since_tweet_id)
self.publisher.save_status_associations()
except Exception as e:
lgt("Encountered error after " +
str(config.MASTODON_RETRIES) +
" retries. Not retrying.")
print(e)
# From times to times we update the Twitter URL length.
self.publisher.update_twitter_link_length()
from mtt import config
from mtt.credentials import check_credentials, setup_credentials
from mtt.mastodon_to_twitter import TwitterPublisher
from mtt.twitter_to_mastodon import MastodonPublisher
from mtt.utils import lgt
#
# First step: check credentials
#
if not check_credentials():
setup_credentials()
lgt('Everything looks good; starting…')
#
# Read in credentials
#
with config.FILES['credentials_twitter'].open('r') as secret_file:
TWITTER_CONSUMER_KEY = secret_file.readline().rstrip()
TWITTER_CONSUMER_SECRET = secret_file.readline().rstrip()
TWITTER_ACCESS_KEY = secret_file.readline().rstrip()
TWITTER_ACCESS_SECRET = secret_file.readline().rstrip()
with config.FILES['credentials_mastodon_server'].open('r') as secret_file:
MASTODON_BASE_URL = secret_file.readline().rstrip()