def test_calc_expected_status_length_with_wide_unicode(self):
status = "…"
len_status = calc_expected_status_length(status)
assert len_status == 2
status = "……"
len_status = calc_expected_status_length(status)
assert len_status == 4
def format_tweets(payload):
"""
Split plain text content into a list.
The length of every element can't be higher than CHARACTER_LIMIT (280 characters).
The formatting rules defined here are:
1. The first element (even if there's only one) must include its attribution (link and author) at the end.
It must be at the end because Twitter only generated a card preview for the last link in a tweet.
2. If there's more than 1 tweet, all the tweets include a placeholder ([…]) to indicate the continuation,
except the last one.
For more examples take a look at the test suite.
"""
content = payload["plain"]
content = " ".join(content.split())
content = unescape(content)
ending = """ {link} shared by @{username}""".format(
link=payload["link"], username=payload["twitter_username"])
if "hashtags" in payload and payload["hashtags"]:
# hashtags: "#hash1,#hash2"
# result: "#hash1 #hash2"
ending = ending + " " + payload["hashtags"].replace(" ", "").replace(
",", " ")
placeholder = " […]"
current_line_length = 0
line = []
tweets = []
words = re.split(r"\s", content)
for word in words:
current_line_length += word_length(word) + 1
if len(tweets) == 0:
if current_line_length > (
CHARACTER_LIMIT -
calc_expected_status_length(placeholder + ending)):
tweets.append(" ".join(line) + placeholder + ending)
line = [word]
current_line_length = word_length(word)
else:
line.append(word)
else:
if current_line_length > (
CHARACTER_LIMIT -
calc_expected_status_length(placeholder)):
tweets.append(" ".join(line) + placeholder)
line = [word]
current_line_length = word_length(word)
else:
line.append(word)
if len(tweets) == 0:
tweets.append(" ".join(line) + ending)
else:
tweets.append(" ".join(line))
return tweets
def truncate_status(status):
if twitter_utils.calc_expected_status_length(status) <= 280:
return status
else:
while twitter_utils.calc_expected_status_length(status) > 278:
status = status[:len(status)-1]
status += '…'
return status
def thread_parser(s, **options):
"""
Splits the thread by delimiter + \n, then processes that thread by parsing out media paths
:type s: str
:rtype: List[status object]
"""
base_parsed_thread = s.split(options['d'] + '\n')
base_parsed_thread = list(map(lambda e: e.strip(), base_parsed_thread))
status = []
for tweet in base_parsed_thread:
if len(tweet) > 0:
status.append(tweet_parser(tweet))
#debug
#print (calc_expected_status_length(tweet))
invalid_lengths = list(
filter(lambda e: calc_expected_status_length(e.tweet) > 280, status))
if len(invalid_lengths) != 0:
for bad_tweet in invalid_lengths:
print(bad_tweet.tweet)
raise Exception('Above tweets have invalid lengths')
return status
async def split_tweet_content(self, content, counter=None, counter_max=None, tweet_id=None):
# add twitter counter at the beginning
if counter is not None and counter_max is not None:
content = f"{counter}/{counter_max} {content}"
counter += 1
# get the current content size
post_size = calc_expected_status_length(content)
# check if the current content size can be posted
if post_size > CHARACTER_LIMIT:
# calculate the number of post required for the whole content
if not counter_max:
counter_max = post_size // CHARACTER_LIMIT
counter = 1
# post the current tweet
post = self.twitter_api.PostUpdate(status=content[:CHARACTER_LIMIT], in_reply_to_status_id=tweet_id)
# recursive call for all post while content > CHARACTER_LIMIT
await self.split_tweet_content(content[CHARACTER_LIMIT:],
counter=counter,
counter_max=counter_max,
tweet_id=tweet_id)
return post
else:
return self.twitter_api.PostUpdate(status=content[:CHARACTER_LIMIT], in_reply_to_status_id=tweet_id)
def test_tweet(self):
consumer_key = settings.TWITTER_CONSUMER_KEY
consumer_secret = settings.TWITTER_CONSUMER_SECRET
access_key = settings.TWITTER_ACCESS_TOKEN_KEY
access_secret = settings.TWITTER_ACCESS_TOKEN_SECRET
if any(not i for i in [
consumer_key,
consumer_secret,
access_key,
access_secret,
]):
LOG.warning("Twitter API credentials not set!")
return
api = ThreadedApi(
consumer_key=consumer_key,
consumer_secret=consumer_secret,
access_token_key=access_key,
access_token_secret=access_secret,
)
message = "This is a test long tweet\r\n{}".format("\r\n".join(
f"This is line {line}" for line in range(2, 30)))
try:
if calc_expected_status_length(message) > CHARACTER_LIMIT:
api.PostUpdates(message, continuation="…", threaded=True)
else:
api.PostUpdate(message)
except TwitterError as exc:
LOG.warning("Hit twitter error: %s", exc)
if str(exc) in RETRYABLE_ERRORS:
raise self.retry(exc=exc)
delay = get_twitter_rate_limit_delay(api)
if delay is None:
LOG.error("No idea what to do with twitter error %s", exc)
raise
raise self.retry(countdown=delay, exc=exc)
def send_tweet_entrance(message, file=None):
if file is None:
# normal text tweet, could be too long
text = message.text
if calc_expected_status_length(text) > CHARACTER_LIMIT:
# pre calculation
is_long = False
for part in re.split(r"\n+", message.text):
if calc_expected_status_length(part) > CHARACTER_LIMIT:
logging.warning("%s chars for thread tweet!",
calc_expected_status_length(part))
bot.reply_to(message,
f"`{part}` is too long",
parse_mode="markdown")
is_long = True
if is_long:
return
# normal process
url = ""
first_thread = []
for part in re.split(r"\n+", message.text):
new_message = copy.copy(message)
if url:
replied = types.Message.de_json(reply_json % url)
setattr(new_message, "reply_to_message", replied)
new_message.text = part
result = send_tweet(new_message)
first_thread.append(result)
url = tweet_format.format(screen_name="abc", id=result["id"])
# send response
send_tweet_message(first_thread[0], message)
return
else:
result = send_tweet(message)
elif isinstance(file, list):
result = send_tweet(message, file)
[f.close() for f in file]
else:
# one image message, file object
result = send_tweet(message, [file])
file.close()
send_tweet_message(result, message)
def tweet_too_big(tweet):
'''
Verify if the message respects Twitter's limitation of characters
:param tweet: (string) the message the user wants to publish on Twitter
:return: True if the message is longer than 280 characters, False otherwise
'''
if twitter_utils.calc_expected_status_length(tweet) > 280:
return True
else:
return False
def split_tweet(tweet, start_sep=TWEET_TBC, end_sep=TWEET_TBC):
toks = tweet.split(' ')
safety_len = len(end_sep) + 5
cur_tweet = ''
prev_tok = None
tweets = []
for tok in toks:
extended_tweet = cur_tweet
if prev_tok is not None and prev_tok != '\n':
extended_tweet += ' '
extended_tweet += tok
prev_tok = tok
if twitter_utils.calc_expected_status_length(extended_tweet) + safety_len > MAX_TWEET_LEN:
tweets.append(cur_tweet + end_sep)
cur_tweet = start_sep + tok
else:
cur_tweet = extended_tweet
tweets.append(cur_tweet)
return tweets
def send_tweets(tweets):
"""
Publish tweets as a thread.
"""
total = len(tweets)
for i, tweet in enumerate(tweets):
length = calc_expected_status_length(tweet)
nro = i + 1
print("Tweet {nro}/{total} ({length} chars):\n{tweet}".format(
nro=nro, total=total, length=length, tweet=tweet))
api = twitter.Api(
consumer_key=CONSUMER_KEY,
consumer_secret=CONSUMER_SECRET,
access_token_key=ACCESS_TOKEN_KEY,
access_token_secret=ACCESS_TOKEN_SECRET,
)
last_reply_to_id = None
for tweet in tweets:
status = api.PostUpdate(tweet, in_reply_to_status_id=last_reply_to_id)
last_reply_to_id = status.id
return
def remain_char(tweet: str) -> str:
length = calc_expected_status_length(tweet)
return f"{length}/{CHARACTER_LIMIT}. Click this message to tweet."
def tweet(self):
""" Create message, reduce it until it fits in a tweet, and then tweet
it with a link to Google maps and tweet location included.
"""
def generate_tag_string(hashtags):
'''create hashtag string'''
tag_string = ''
if hashtags:
for hashtag in hashtags:
tag_string += ' #{}'.format(hashtag)
return tag_string
try:
api = twitter.Api(consumer_key=config.TWITTER_CONSUMER_KEY,
consumer_secret=config.TWITTER_CONSUMER_SECRET,
access_token_key=config.TWITTER_ACCESS_KEY,
access_token_secret=config.TWITTER_ACCESS_SECRET)
except Exception:
self.logger.exception('Failed to create a Twitter API object.')
tag_string = generate_tag_string(self.hashtags)
if self.expire_time:
tweet_text = (
'A {d} {n} appeared! It will be {p} until {e}. {t} {u}'
).format(d=self.description,
n=self.name,
p=self.place,
e=self.expire_time,
t=tag_string,
u=self.map_link)
else:
tweet_text = (
'A {d} {n} appeared {p}! It will expire sometime between '
'{e1} and {e2}. {t} {u}').format(d=self.description,
n=self.name,
p=self.place,
e1=self.min_expire_time,
e2=self.max_expire_time,
t=tag_string,
u=self.map_link)
if calc_expected_status_length(tweet_text) > 140:
tweet_text = tweet_text.replace(' meters ', 'm ')
# remove hashtags until length is short enough
while calc_expected_status_length(tweet_text) > 140:
if self.hashtags:
hashtag = self.hashtags.pop()
tweet_text = tweet_text.replace(' #' + hashtag, '')
else:
break
if (calc_expected_status_length(tweet_text) > 140
and self.landmark.shortname):
tweet_text = tweet_text.replace(self.landmark.name,
self.landmark.shortname)
if calc_expected_status_length(tweet_text) > 140:
place = self.landmark.shortname or self.landmark.name
phrase = self.landmark.phrase
if self.place.startswith(phrase):
place_string = '{ph} {pl}'.format(ph=phrase, pl=place)
else:
place_string = 'near {}'.format(place)
tweet_text = tweet_text.replace(self.place, place_string)
if calc_expected_status_length(tweet_text) > 140:
if self.expire_time:
tweet_text = 'A {d} {n} will be {p} until {e}. {u}'.format(
d=self.description,
n=self.name,
p=place_string,
e=self.expire_time,
u=self.map_link)
else:
tweet_text = (
"A {d} {n} appeared {p}! It'll expire between {e1} & {e2}."
' {u}').format(d=self.description,
n=self.name,
p=place_string,
e1=self.min_expire_time,
e2=self.max_expire_time,
u=self.map_link)
if calc_expected_status_length(tweet_text) > 140:
if self.expire_time:
tweet_text = 'A {d} {n} will expire at {e}. {u}'.format(
n=self.name, e=self.expire_time, u=self.map_link)
else:
tweet_text = (
'A {d} {n} will expire between {e1} & {e2}. {u}').format(
d=self.description,
n=self.name,
e1=self.min_expire_time,
e2=self.max_expire_time,
u=self.map_link)
image = None
if config.TWEET_IMAGES:
try:
image = PokeImage(self.pokemon_id, self.iv, self.moves,
self.time_of_day).create()
except Exception:
self.logger.exception('Failed to create a Tweet image.')
try:
api.PostUpdate(tweet_text,
media=image,
latitude=self.coordinates[0],
longitude=self.coordinates[1],
display_coordinates=True)
except Exception:
self.logger.exception('Failed to Tweet about {}.'.format(
self.name))
return False
else:
self.logger.info('Sent a tweet about {}.'.format(self.name))
return True
finally:
try:
image.close()
except AttributeError:
pass
def test_calc_expected_status_length_with_url_and_extra_spaces(self):
status = 'hi a tweet there example.com'
len_status = calc_expected_status_length(status)
self.assertEqual(len_status, 63)
def tweet_about_beers(self, beer_pks):
if not beer_pks:
LOG.warning("nothing to do")
return
LOG.debug("Tweeting about beer PKs: %s", beer_pks)
consumer_key = settings.TWITTER_CONSUMER_KEY
consumer_secret = settings.TWITTER_CONSUMER_SECRET
access_key = settings.TWITTER_ACCESS_TOKEN_KEY
access_secret = settings.TWITTER_ACCESS_TOKEN_SECRET
if any(not i for i in [
consumer_key,
consumer_secret,
access_key,
access_secret,
]):
LOG.warning("Twitter API credentials not set!")
return
api = ThreadedApi(
consumer_key=consumer_key,
consumer_secret=consumer_secret,
access_token_key=access_key,
access_token_secret=access_secret,
)
# Mark beers which have been removed from the tap list as tweeted about
Beer.objects.filter(
tweeted_about=False,
taps__isnull=True,
).update(tweeted_about=True)
beers = list(
Beer.objects.filter(
id__in=beer_pks,
tweeted_about=False,
).select_related(
"manufacturer",
"style",
).prefetch_related(
Prefetch(
"taps",
queryset=Tap.objects.select_related("venue"),
), ).order_by("id"))
already_tweeted_about = set(i.id for i in Beer.objects.filter(
tweeted_about=True,
id__in=beer_pks,
))
unknown_pks = set(beer_pks).difference(already_tweeted_about)
LOG.debug("Got %s beers", len(beers))
if not beers:
if unknown_pks:
LOG.warning("No beers found! Trying again shortly")
raise self.retry(countdown=300)
LOG.info("everything was already tweeted about. No big deal")
return
if len(beers) > 10:
LOG.info("Too many beers to tweet about at once: %s", len(beers))
beers = beers[:10]
if len(beers) == 1:
beer = beers[0]
if beer.tweeted_about:
LOG.info("%s has already been tweeted about; skipping.", beer)
return
message = format_beer(beer, SINGLE_BEER_TEMPLATE).strip()
messages = [message]
else:
extra_beers = len(beer_pks) - len(beers) - len(already_tweeted_about)
messages = [
MULTI_BEER_OUTER.format(
len(beers),
"({} still to come!)".format(extra_beers)
if extra_beers > 0 else "",
).strip()
] + format_beers(beers)
if len(messages) == 1:
LOG.info("All beers already tweeted about")
return
message = "\r\n".join(messages)
LOG.info("Going to tweet: %s", message)
try:
if calc_expected_status_length(message) > CHARACTER_LIMIT:
api.PostUpdates(message, continuation="…", threaded=True)
else:
api.PostUpdate(message)
except TwitterError as exc:
LOG.warning("Hit twitter error: %s", exc)
if str(exc) in RETRYABLE_ERRORS:
raise self.retry(exc=exc)
LOG.error("Tweet(s) that caused error was %s", message)
delay = get_twitter_rate_limit_delay(api)
if delay is None:
LOG.error("No idea what to do with twitter error %s", exc)
raise
raise self.retry(countdown=delay, exc=exc)
Beer.objects.filter(id__in=[i.id
for i in beers]).update(tweeted_about=True)
LOG.debug("Done tweeting")
def tweet_about_beers(self, beer_pks):
if not beer_pks:
LOG.warning('nothing to do')
return
LOG.debug('Tweeting about beer PKs: %s', beer_pks)
consumer_key = settings.TWITTER_CONSUMER_KEY
consumer_secret = settings.TWITTER_CONSUMER_SECRET
access_key = settings.TWITTER_ACCESS_TOKEN_KEY
access_secret = settings.TWITTER_ACCESS_TOKEN_SECRET
if any(not i for i in [
consumer_key,
consumer_secret,
access_key,
access_secret,
]):
LOG.warning('Twitter API credentials not set!')
return
api = ThreadedApi(
consumer_key=consumer_key,
consumer_secret=consumer_secret,
access_token_key=access_key,
access_token_secret=access_secret,
)
beers = list(
Beer.objects.filter(
id__in=beer_pks,
tweeted_about=False,
).select_related(
'manufacturer',
'style',
).prefetch_related(
Prefetch(
'taps',
queryset=Tap.objects.select_related('venue'),
), ).order_by('id'))
already_tweeted_about = set(i.id for i in Beer.objects.filter(
tweeted_about=True,
id__in=beer_pks,
))
unknown_pks = set(beer_pks).difference(already_tweeted_about)
LOG.debug('Got %s beers', len(beers))
if not beers:
if unknown_pks:
LOG.warning('No beers found! Trying again shortly')
raise self.retry(countdown=300)
LOG.info('everything was already tweeted about. No big deal')
return
if len(beers) > 10:
LOG.info('Too many beers to tweet about at once: %s', len(beers))
beers = beers[:10]
if len(beers) == 1:
beer = beers[0]
if beer.tweeted_about:
LOG.info('%s has already been tweeted about; skipping.', beer)
return
message = format_beer(beer, SINGLE_BEER_TEMPLATE).strip()
messages = [message]
else:
extra_beers = len(beer_pks) - len(beers) - len(already_tweeted_about)
messages = [
MULTI_BEER_OUTER.format(
len(beers), '({} still to come!)'.format(extra_beers)
if extra_beers > 0 else '').strip()
] + format_beers(beers)
if len(messages) == 1:
LOG.info('All beers already tweeted about')
return
message = '\r\n'.join(messages)
LOG.info('Going to tweet: %s', message)
try:
if calc_expected_status_length(message) > CHARACTER_LIMIT:
api.PostUpdates(message, continuation='…', threaded=True)
else:
api.PostUpdate(message)
except TwitterError as exc:
LOG.warning('Hit twitter error: %s', exc)
if str(exc) in RETRYABLE_ERRORS:
raise self.retry(exc=exc)
delay = get_twitter_rate_limit_delay(api)
if delay is None:
LOG.error('No idea what to do with twitter error %s', exc)
raise
raise self.retry(countdown=delay, exc=exc)
Beer.objects.filter(id__in=[i.id
for i in beers]).update(tweeted_about=True)
LOG.debug('Done tweeting')
def test_calc_expected_status_length_with_url_and_extra_spaces(self):
status = 'hi a tweet there example.com'
len_status = calc_expected_status_length(status)
self.assertEqual(len_status, 63)
def test_calc_expected_status_length_with_url(self):
status = 'hi a tweet there example.com'
len_status = calc_expected_status_length(status)
self.assertEqual(len_status, 40)
def test_calc_expected_status_length(self):
status = 'hi a tweet there'
len_status = calc_expected_status_length(status)
self.assertEqual(len_status, 16)
def test_calc_expected_status_length_with_url(self):
status = 'hi a tweet there example.com'
len_status = calc_expected_status_length(status)
self.assertEqual(len_status, 40)
def test_calc_expected_status_length(self):
status = 'hi a tweet there'
len_status = calc_expected_status_length(status)
self.assertEqual(len_status, 16)