def guess_raw_share_tweet_content(post):
preview = ''
if not post.repost_contexts:
current_app.logger.debug(
'failed to load repost context for %s', post.id)
return None
context = post.repost_contexts[0]
if context.title:
preview += context.title
if context.author_name:
preview += ' by ' + context.author_name
elif context.content:
if context.author_name:
preview += context.author_name + ': '
preview += context.content_plain
# if the tweet doesn't get trimmed, put the link on the end anyway
preview += (' ' if preview else '') + context.permalink
target_length = TWEET_CHAR_LENGTH
imgs = list(collect_images(context))
img_url = imgs[0] if imgs else None
preview = brevity.shorten(preview, permalink=context.permalink,
target_length=target_length)
return preview, img_url
def guess_raw_share_tweet_content(post):
preview = ''
if not post.repost_contexts:
current_app.logger.debug('failed to load repost context for %s',
post.id)
return None
context = post.repost_contexts[0]
if context.title:
preview += context.title
if context.author_name:
preview += ' by ' + context.author_name
elif context.content:
if context.author_name:
preview += context.author_name + ': '
preview += context.content_plain
# if the tweet doesn't get trimmed, put the link on the end anyway
preview += (' ' if preview else '') + context.permalink
target_length = TWEET_CHAR_LENGTH
imgs = list(collect_images(context))
img_url = imgs[0] if imgs else None
preview = brevity.shorten(preview,
permalink=context.permalink,
target_length=target_length)
return preview, img_url
def _truncate(self, content, url, include_link, type, has_media):
"""Shorten tweet content to fit within the 140 character limit.
Args:
content: string
url: string
include_link: string
type: string
has_media: boolean
Return: string, the possibly shortened and ellipsized tweet text
"""
if type == 'article':
format = brevity.FORMAT_ARTICLE
else:
format = brevity.FORMAT_NOTE
if has_media:
format += '+' + brevity.FORMAT_MEDIA
return brevity.shorten(
content,
# permalink is included only when the text is truncated
permalink=url if include_link != source.OMIT_LINK else None,
# permashortlink is always included
permashortlink=url if include_link == source.INCLUDE_LINK else None,
target_length=MAX_TWEET_LENGTH, link_length=TCO_LENGTH, format=format)
def test_no_shorten_with_cc_tlds(self):
text = 'Despite names,\nind.ie&indie.vc are NOT #indieweb @indiewebcamp\nindiewebcamp.com/2014-review#Indie_Term_Re-use\n@iainspad @sashtown @thomatronic'
permalink = 'http://tantek.com/2015/013/t1/names-ind-ie-indie-vc-not-indieweb'
psc = 'ttk.me t4_81'
result = brevity.shorten(text=text, permalink=permalink,
permashortcitation=psc)
self.assertEqual('{} ({})'.format(text, psc), result)
def test_mmddyyyy_false_positive(self):
text = u'anybody have a wedding ring with the date engraved in ISO 8601? I’ll be damned if I’m going to wear mm.dd.yyyy anywhere on my person.'
permalink = 'https://kylewm.com/2015/05/anybody-have-a-wedding-ring-with-the-date-engraved'
result = brevity.shorten(text=text, permalink=permalink,
permashortlink=None)
self.assertEqual(text, result)
def test_no_shorten_technorati(self):
text = 'Despite Technorati dumping tag & blog search long ago, rel-tag succeeded on web, in #HTML spec https://html.spec.whatwg.org/multipage/semantics.html#linkTypes'
permalink = 'http://tantek.com/2015/014/t3/rel-tag-succeeded-web-html-specs'
psc = 'ttk.me t4_93'
result = brevity.shorten(text=text, permalink=permalink,
permashortcitation=psc)
self.assertEqual('{} ({})'.format(text, psc), result)
def test_hamburg_tld(self):
text = u'ix freue mich auf die nebenan.hamburg morgen. ich spreche auch ne halbe stunde übers #indieweb und @reclaim_fm.'
permalink = u'http://wirres.net/article/articleview/7773/1/6/'
psl = u'http://wirres.net/7773'
expected = u'ix freue mich auf die nebenan.hamburg morgen. ich spreche auch ne halbe stunde übers #indieweb und… http://wirres.net/article/articleview/7773/1/6/'
result = brevity.shorten(text=text, permalink=permalink,
permashortlink=psl)
self.assertEqual(expected, result)
def test_shorten_coming_storm(self):
text = 'Hey #indieweb, the coming storm of webmention Spam may not be far away. Those of us that have input fields to send send webmentions manually may already be getting them. Look at the mentions on http://aaronparecki.com/articles/2015/01/22/1/why-not-json'
permalink = 'https://ben.thatmustbe.me/note/2015/1/31/1/'
psl = 'http://btmb.me/s/6q'
expected = u'Hey #indieweb, the coming storm of webmention Spam may not be far away. Those of us that have input fields to send… ' + permalink
result = brevity.shorten(text=text, permalink=permalink,
permashortlink=psl)
self.assertEqual(expected, result)
def test_shorten(self):
for testcase in TESTS['shorten']:
params = dict([
(k, testcase[k]) for k in (
'text', 'permalink', 'permashortlink', 'permashortcitation',
'target_length', 'link_length', 'format',
)
if k in testcase])
result = brevity.shorten(**params)
expected = testcase['expected']
self.assertEqual(expected, result)
def truncate(self, content, url, include_link, type=None, quote_url=None):
"""Shorten text content to fit within a character limit.
Character limit and URL character length are taken from the
TRUNCATE_TEXT_LENGTH and TRUNCATE_URL_LENGTH class constants
Args:
content: string
url: string
include_link: string
type: string, optional: 'article', 'note', etc.
quote_url: string URL, optional. If provided, it will be appended to the
content, *after* truncating.
Return: string, the possibly shortened and ellipsized text
"""
kwargs = {}
if quote_url:
kwargs['target_length'] = (
(self.TRUNCATE_TEXT_LENGTH
or brevity.WEIGHTS['maxWeightedTweetLength']) -
(self.TRUNCATE_URL_LENGTH
or brevity.WEIGHTS['transformedURLLength']) - 1)
elif self.TRUNCATE_TEXT_LENGTH is not None:
kwargs['target_length'] = self.TRUNCATE_TEXT_LENGTH
if self.TRUNCATE_URL_LENGTH is not None:
kwargs['link_length'] = self.TRUNCATE_URL_LENGTH
if include_link != OMIT_LINK:
kwargs['permalink'] = url # only include when text is truncated
if include_link == INCLUDE_LINK:
kwargs['permashortlink'] = url # always include
if type == 'article':
kwargs['format'] = brevity.FORMAT_ARTICLE
truncated = brevity.shorten(content, **kwargs)
if quote_url:
truncated += ' ' + quote_url
return truncated
def _truncate(self, content, url, include_link, type):
"""Shorten tweet content to fit within the 140 character limit.
Args:
content: string
url: string
include_link: string
type: string: 'article', 'note', etc.
Return: string, the possibly shortened and ellipsized tweet text
"""
if type == 'article':
format = brevity.FORMAT_ARTICLE
else:
format = brevity.FORMAT_NOTE
return brevity.shorten(
content,
# permalink is included only when the text is truncated
permalink=url if include_link != source.OMIT_LINK else None,
# permashortlink is always included
permashortlink=url if include_link == source.INCLUDE_LINK else None,
target_length=MAX_TWEET_LENGTH, link_length=TCO_LENGTH, format=format)
def publish(site):
auth = OAuth1(client_key=current_app.config['TWITTER_CLIENT_KEY'],
client_secret=current_app.config['TWITTER_CLIENT_SECRET'],
resource_owner_key=site.account.token,
resource_owner_secret=site.account.token_secret)
def interpret_response(result):
if result.status_code // 100 != 2:
return util.wrap_silo_error_response(result)
result_json = result.json()
twitter_url = 'https://twitter.com/{}/status/{}'.format(
result_json.get('user', {}).get('screen_name'),
result_json.get('id_str'))
return util.make_publish_success_response(twitter_url, result_json)
def get_tweet_id(original):
tweet_url = util.posse_post_discovery(original, TWEET_RE)
if tweet_url:
m = TWEET_RE.match(tweet_url)
if m:
return m.group(1), m.group(2)
return None, None
def upload_photo(photo):
current_app.logger.debug('uploading photo, name=%s, type=%s',
photo.filename, photo.content_type)
result = requests.post(UPLOAD_MEDIA_URL,
files={
'media': (photo.filename, photo.stream,
photo.content_type),
},
auth=auth)
if result.status_code // 100 != 2:
return None, result
result_data = result.json()
current_app.logger.debug('upload result: %s', result_data)
return result_data.get('media_id_string'), None
def upload_video(video, default_content_type='video/mp4'):
# chunked video upload
chunk_files = []
def cleanup():
for f in chunk_files:
os.unlink(f)
chunk_size = 1 << 20
total_size = 0
while True:
chunk = video.read(chunk_size)
if not chunk:
break
total_size += len(chunk)
tempfd, tempfn = tempfile.mkstemp(
'-%03d-%s' % (len(chunk_files), video.filename))
with open(tempfn, 'wb') as f:
f.write(chunk)
chunk_files.append(tempfn)
current_app.logger.debug('init upload. type=%s, length=%s',
video.content_type, video.content_length)
result = requests.post(UPLOAD_MEDIA_URL,
data={
'command': 'INIT',
'media_type': video.content_type
or default_content_type,
'total_bytes': total_size,
},
auth=auth)
current_app.logger.debug('init result: %s %s', result, result.text)
if result.status_code // 100 != 2:
cleanup()
return None, result
result_data = result.json()
media_id = result_data.get('media_id_string')
segment_idx = 0
for chunk_file in chunk_files:
current_app.logger.debug('appending file: %s', chunk_file)
result = requests.post(UPLOAD_MEDIA_URL,
data={
'command': 'APPEND',
'media_id': media_id,
'segment_index': segment_idx,
},
files={
'media': open(chunk_file, 'rb'),
},
auth=auth)
current_app.logger.debug('append result: %s %s', result,
result.text)
if result.status_code // 100 != 2:
cleanup()
return None, result
segment_idx += 1
current_app.logger.debug('finalize uploading video: %s', media_id)
result = requests.post(UPLOAD_MEDIA_URL,
data={
'command': 'FINALIZE',
'media_id': media_id,
},
auth=auth)
current_app.logger.debug('finalize result: %s %s', result, result.text)
if result.status_code // 100 != 2:
cleanup()
return None, result
cleanup()
return media_id, None
data = {}
format = brevity.FORMAT_NOTE
content = request.form.get('content[value]') or request.form.get('content')
if 'name' in request.form:
format = brevity.FORMAT_ARTICLE
content = request.form.get('name')
repost_ofs = util.get_possible_array_value(request.form, 'repost-of')
for repost_of in repost_ofs:
_, tweet_id = get_tweet_id(repost_of)
if tweet_id:
return interpret_response(
requests.post(RETWEET_STATUS_URL.format(tweet_id), auth=auth))
else:
if repost_ofs:
content = 'Reposted: {}'.format(repost_ofs[0])
like_ofs = util.get_possible_array_value(request.form, 'like-of')
for like_of in like_ofs:
_, tweet_id = get_tweet_id(like_of)
if tweet_id:
return interpret_response(
requests.post(FAVE_STATUS_URL,
data={'id': tweet_id},
auth=auth))
else:
if like_ofs:
content = 'Liked: {}'.format(like_ofs[0])
media_ids = []
for photo in util.get_files_or_urls_as_file_storage(
request.files, request.form, 'photo'):
media_id, err = upload_photo(photo)
if err:
return util.wrap_silo_error_response(err)
media_ids.append(media_id)
for video in util.get_files_or_urls_as_file_storage(
request.files, request.form, 'video'):
media_id, err = upload_video(video)
if err:
return util.wrap_silo_error_response(err)
media_ids.append(media_id)
in_reply_tos = util.get_possible_array_value(request.form, 'in-reply-to')
for in_reply_to in in_reply_tos:
twitterer, tweet_id = get_tweet_id(in_reply_to)
if tweet_id:
data['in_reply_to_status_id'] = tweet_id
break
else:
if in_reply_tos:
content = 'Re: {}, {}'.format(in_reply_tos[0], content)
location = request.form.get('location')
current_app.logger.debug('received location param: %s', location)
data['lat'], data['long'] = util.parse_geo_uri(location)
permalink_url = request.form.get('url')
if media_ids:
data['media_ids'] = ','.join(media_ids)
if content:
data['status'] = brevity.shorten(content,
permalink=permalink_url,
format=format,
target_length=280)
# for in-reply-to tweets, leading @mentions will be looked up from the original Tweet, and added to the new Tweet from there.
# https://developer.twitter.com/en/docs/tweets/post-and-engage/api-reference/post-statuses-update
data['auto_populate_reply_metadata'] = 'true'
data = util.trim_nulls(data)
current_app.logger.debug('publishing with params %s', data)
return interpret_response(
requests.post(CREATE_STATUS_URL, data=data, auth=auth))
def guess_tweet_content(post, in_reply_to):
"""Best guess effort to generate tweet content for a post; useful for
auto-filling the share form.
"""
preview = ''
if post.title:
preview += post.title
# add location if it's a checkin
elif post.post_type == 'checkin' and post.venue:
preview = 'Checked in to ' + post.venue.name
text_content = format_markdown_as_tweet(post.content)
if text_content:
preview += (': ' if preview else '') + text_content
# add an in-reply-to if one isn't there already
if in_reply_to:
reply_match = PERMALINK_RE.match(in_reply_to)
if reply_match:
# get the status we're responding to
status_response = requests.get(
'https://api.twitter.com/1.1/statuses/show/{}.json'.format(
reply_match.group(2)),
auth=get_auth())
if status_response.status_code // 2 != 100:
current_app.logger.warn(
'failed to fetch tweet %s %s while finding participants',
status_response, status_response.content)
status_data = {}
else:
status_data = status_response.json()
# get the list of people to respond to
mentioned_users = []
my_screen_name = get_authed_twitter_account().get(
'screen_name', '')
for user in status_data.get('entities',
{}).get('user_mentions', []):
screen_name = user.get('screen_name', '')
if screen_name and screen_name.lower() != my_screen_name.lower(
):
mentioned_users.append(screen_name)
mentioned_users.append(reply_match.group(1)) # the status author
current_app.logger.debug('got mentioned users %s', mentioned_users)
# check to see if anybody is already mentioned by the preview
mention_match = USERMENTION_RE.findall(preview)
for match in mention_match:
if match[0] in mentioned_users:
break
else:
# nobody was mentioned, prepend all the names!
for user in mentioned_users:
preview = prepend_twitter_name(user, preview)
target_length = TWEET_CHAR_LENGTH
img_url = None
if post.post_type == 'photo' and post.attachments:
img_url = post.attachments[0].url
target_length -= MEDIA_CHAR_LENGTH
preview = brevity.shorten(preview,
permalink=post.permalink,
target_length=target_length)
return preview, img_url
def guess_tweet_content(post, in_reply_to):
"""Best guess effort to generate tweet content for a post; useful for
auto-filling the share form.
"""
preview = ''
if post.title:
preview += post.title
# add location if it's a checkin
elif post.post_type == 'checkin' and post.venue:
preview = 'Checked in to ' + post.venue.name
text_content = format_markdown_as_tweet(post.content)
if text_content:
preview += (': ' if preview else '') + text_content
# add an in-reply-to if one isn't there already
if in_reply_to:
reply_match = PERMALINK_RE.match(in_reply_to)
if reply_match:
# get the status we're responding to
status_response = requests.get(
'https://api.twitter.com/1.1/statuses/show/{}.json'.format(
reply_match.group(2)),
auth=get_auth())
if status_response.status_code // 2 != 100:
current_app.logger.warn(
'failed to fetch tweet %s %s while finding participants',
status_response, status_response.content)
status_data = {}
else:
status_data = status_response.json()
# get the list of people to respond to
mentioned_users = []
my_screen_name = get_authed_twitter_account().get(
'screen_name', '')
for user in status_data.get('entities', {}).get('user_mentions', []):
screen_name = user.get('screen_name', '')
if screen_name and screen_name.lower() != my_screen_name.lower():
mentioned_users.append(screen_name)
mentioned_users.append(reply_match.group(1)) # the status author
current_app.logger.debug('got mentioned users %s', mentioned_users)
# check to see if anybody is already mentioned by the preview
mention_match = USERMENTION_RE.findall(preview)
for match in mention_match:
if match[0] in mentioned_users:
break
else:
# nobody was mentioned, prepend all the names!
for user in mentioned_users:
preview = prepend_twitter_name(user, preview)
target_length = TWEET_CHAR_LENGTH
img_url = None
if post.post_type == 'photo' and post.attachments:
img_url = post.attachments[0].url
target_length -= MEDIA_CHAR_LENGTH
preview = brevity.shorten(preview, permalink=post.permalink,
target_length=target_length)
return preview, img_url
def test_no_shorten_intl_characters(self):
text = u"Si Hären Engel duurch all, Haus Benn dé blo, am wuel Kolrettchen Nuechtegall dén. Nun en schéi Milliounen, an wee drem d'Welt, do Ierd blénk"
self.assertEqual(text, brevity.shorten(text=text))
def publish(site):
auth = OAuth1(
client_key=current_app.config['TWITTER_CLIENT_KEY'],
client_secret=current_app.config['TWITTER_CLIENT_SECRET'],
resource_owner_key=site.account.token,
resource_owner_secret=site.account.token_secret)
def interpret_response(result):
if result.status_code // 100 != 2:
return util.wrap_silo_error_response(result)
result_json = result.json()
twitter_url = 'https://twitter.com/{}/status/{}'.format(
result_json.get('user', {}).get('screen_name'),
result_json.get('id_str'))
return util.make_publish_success_response(twitter_url, result_json)
def get_tweet_id(original):
tweet_url = util.posse_post_discovery(original, TWEET_RE)
if tweet_url:
m = TWEET_RE.match(tweet_url)
if m:
return m.group(1), m.group(2)
return None, None
def upload_photo(photo):
current_app.logger.debug('uploading photo, name=%s, type=%s',
photo.filename, photo.content_type)
result = requests.post(UPLOAD_MEDIA_URL, files={
'media': (photo.filename, photo.stream, photo.content_type),
}, auth=auth)
if result.status_code // 100 != 2:
return None, result
result_data = result.json()
current_app.logger.debug('upload result: %s', result_data)
return result_data.get('media_id_string'), None
def upload_video(video, default_content_type='video/mp4'):
# chunked video upload
chunk_files = []
def cleanup():
for f in chunk_files:
os.unlink(f)
chunk_size = 1 << 20
total_size = 0
while True:
chunk = video.read(chunk_size)
if not chunk:
break
total_size += len(chunk)
tempfd, tempfn = tempfile.mkstemp('-%03d-%s' % (
len(chunk_files), video.filename))
with open(tempfn, 'wb') as f:
f.write(chunk)
chunk_files.append(tempfn)
current_app.logger.debug('init upload. type=%s, length=%s',
video.content_type, video.content_length)
result = requests.post(UPLOAD_MEDIA_URL, data={
'command': 'INIT',
'media_type': video.content_type or default_content_type,
'total_bytes': total_size,
}, auth=auth)
current_app.logger.debug('init result: %s %s', result, result.text)
if result.status_code // 100 != 2:
cleanup()
return None, result
result_data = result.json()
media_id = result_data.get('media_id_string')
segment_idx = 0
for chunk_file in chunk_files:
current_app.logger.debug('appending file: %s', chunk_file)
result = requests.post(UPLOAD_MEDIA_URL, data={
'command': 'APPEND',
'media_id': media_id,
'segment_index': segment_idx,
}, files={
'media': open(chunk_file, 'rb'),
}, auth=auth)
current_app.logger.debug(
'append result: %s %s', result, result.text)
if result.status_code // 100 != 2:
cleanup()
return None, result
segment_idx += 1
current_app.logger.debug('finalize uploading video: %s', media_id)
result = requests.post(UPLOAD_MEDIA_URL, data={
'command': 'FINALIZE',
'media_id': media_id,
}, auth=auth)
current_app.logger.debug('finalize result: %s %s', result, result.text)
if result.status_code // 100 != 2:
cleanup()
return None, result
cleanup()
return media_id, None
data = {}
format = brevity.FORMAT_NOTE
content = request.form.get('content[value]') or request.form.get('content')
if 'name' in request.form:
format = brevity.FORMAT_ARTICLE
content = request.form.get('name')
repost_ofs = util.get_possible_array_value(request.form, 'repost-of')
for repost_of in repost_ofs:
_, tweet_id = get_tweet_id(repost_of)
if tweet_id:
return interpret_response(
requests.post(RETWEET_STATUS_URL.format(tweet_id), auth=auth))
else:
if repost_ofs:
content = 'Reposted: {}'.format(repost_ofs[0])
like_ofs = util.get_possible_array_value(request.form, 'like-of')
for like_of in like_ofs:
_, tweet_id = get_tweet_id(like_of)
if tweet_id:
return interpret_response(
requests.post(FAVE_STATUS_URL, data={'id': tweet_id}, auth=auth))
else:
if like_ofs:
content = 'Liked: {}'.format(like_ofs[0])
media_ids = []
for photo in util.get_files_or_urls_as_file_storage(request.files, request.form, 'photo'):
media_id, err = upload_photo(photo)
if err:
return util.wrap_silo_error_response(err)
media_ids.append(media_id)
for video in util.get_files_or_urls_as_file_storage(request.files, request.form, 'video'):
media_id, err = upload_video(video)
if err:
return util.wrap_silo_error_response(err)
media_ids.append(media_id)
in_reply_tos = util.get_possible_array_value(request.form, 'in-reply-to')
for in_reply_to in in_reply_tos:
twitterer, tweet_id = get_tweet_id(in_reply_to)
if tweet_id:
data['in_reply_to_status_id'] = tweet_id
break
else:
if in_reply_tos:
content = 'Re: {}, {}'.format(in_reply_tos[0], content)
location = request.form.get('location')
current_app.logger.debug('received location param: %s', location)
data['lat'], data['long'] = util.parse_geo_uri(location)
permalink_url = request.form.get('url')
if media_ids:
data['media_ids'] = ','.join(media_ids)
if content:
data['status'] = brevity.shorten(content, permalink=permalink_url,
format=format, target_length=280)
# for in-reply-to tweets, leading @mentions will be looked up from the original Tweet, and added to the new Tweet from there.
# https://developer.twitter.com/en/docs/tweets/post-and-engage/api-reference/post-statuses-update
data['auto_populate_reply_metadata'] = 'true'
data = util.trim_nulls(data)
current_app.logger.debug('publishing with params %s', data)
return interpret_response(
requests.post(CREATE_STATUS_URL, data=data, auth=auth))