Example #1
0
def guess_raw_share_tweet_content(post):
    preview = ''
    if not post.repost_contexts:
        current_app.logger.debug(
            'failed to load repost context for %s', post.id)
        return None
    context = post.repost_contexts[0]

    if context.title:
        preview += context.title

        if context.author_name:
            preview += ' by ' + context.author_name
    elif context.content:
        if context.author_name:
            preview += context.author_name + ': '

        preview += context.content_plain

    # if the tweet doesn't get trimmed, put the link on the end anyway
    preview += (' ' if preview else '') + context.permalink

    target_length = TWEET_CHAR_LENGTH

    imgs = list(collect_images(context))
    img_url = imgs[0] if imgs else None

    preview = brevity.shorten(preview, permalink=context.permalink,
                              target_length=target_length)
    return preview, img_url
Example #2
0
def guess_raw_share_tweet_content(post):
    preview = ''
    if not post.repost_contexts:
        current_app.logger.debug('failed to load repost context for %s',
                                 post.id)
        return None
    context = post.repost_contexts[0]

    if context.title:
        preview += context.title

        if context.author_name:
            preview += ' by ' + context.author_name
    elif context.content:
        if context.author_name:
            preview += context.author_name + ': '

        preview += context.content_plain

    # if the tweet doesn't get trimmed, put the link on the end anyway
    preview += (' ' if preview else '') + context.permalink

    target_length = TWEET_CHAR_LENGTH

    imgs = list(collect_images(context))
    img_url = imgs[0] if imgs else None

    preview = brevity.shorten(preview,
                              permalink=context.permalink,
                              target_length=target_length)
    return preview, img_url
Example #3
0
  def _truncate(self, content, url, include_link, type, has_media):
    """Shorten tweet content to fit within the 140 character limit.

    Args:
      content: string
      url: string
      include_link: string
      type: string
      has_media: boolean

    Return: string, the possibly shortened and ellipsized tweet text
    """
    if type == 'article':
      format = brevity.FORMAT_ARTICLE
    else:
      format = brevity.FORMAT_NOTE
    if has_media:
      format += '+' + brevity.FORMAT_MEDIA

    return brevity.shorten(
      content,
      # permalink is included only when the text is truncated
      permalink=url if include_link != source.OMIT_LINK else None,
      # permashortlink is always included
      permashortlink=url if include_link == source.INCLUDE_LINK else None,
      target_length=MAX_TWEET_LENGTH, link_length=TCO_LENGTH, format=format)
Example #4
0
    def test_no_shorten_with_cc_tlds(self):
        text = 'Despite names,\nind.ie&indie.vc are NOT #indieweb @indiewebcamp\nindiewebcamp.com/2014-review#Indie_Term_Re-use\n@iainspad @sashtown @thomatronic'
        permalink = 'http://tantek.com/2015/013/t1/names-ind-ie-indie-vc-not-indieweb'
        psc = 'ttk.me t4_81'

        result = brevity.shorten(text=text, permalink=permalink,
                                 permashortcitation=psc)
        self.assertEqual('{} ({})'.format(text, psc), result)
Example #5
0
    def test_mmddyyyy_false_positive(self):
        text = u'anybody have a wedding ring with the date engraved in ISO 8601? I’ll be damned if I’m going to wear mm.dd.yyyy anywhere on my person.'

        permalink = 'https://kylewm.com/2015/05/anybody-have-a-wedding-ring-with-the-date-engraved'

        result = brevity.shorten(text=text, permalink=permalink,
                                 permashortlink=None)
        self.assertEqual(text, result)
Example #6
0
    def test_no_shorten_technorati(self):
        text = 'Despite Technorati dumping tag & blog search long ago, rel-tag succeeded on web, in #HTML spec https://html.spec.whatwg.org/multipage/semantics.html#linkTypes'
        permalink = 'http://tantek.com/2015/014/t3/rel-tag-succeeded-web-html-specs'
        psc = 'ttk.me t4_93'

        result = brevity.shorten(text=text, permalink=permalink,
                                 permashortcitation=psc)
        self.assertEqual('{} ({})'.format(text, psc), result)
Example #7
0
    def test_hamburg_tld(self):
        text = u'ix freue mich auf die nebenan.hamburg morgen. ich spreche auch ne halbe stunde übers #indieweb und @reclaim_fm.'

        permalink = u'http://wirres.net/article/articleview/7773/1/6/'
        psl = u'http://wirres.net/7773'

        expected = u'ix freue mich auf die nebenan.hamburg morgen. ich spreche auch ne halbe stunde übers #indieweb und… http://wirres.net/article/articleview/7773/1/6/'
        result = brevity.shorten(text=text, permalink=permalink,
                                 permashortlink=psl)
        self.assertEqual(expected, result)
Example #8
0
    def test_shorten_coming_storm(self):
        text = 'Hey #indieweb, the coming storm of webmention Spam may not be far away. Those of us that have input fields to send send webmentions manually may already be getting them. Look at the mentions on http://aaronparecki.com/articles/2015/01/22/1/why-not-json'

        permalink = 'https://ben.thatmustbe.me/note/2015/1/31/1/'
        psl = 'http://btmb.me/s/6q'

        expected = u'Hey #indieweb, the coming storm of webmention Spam may not be far away. Those of us that have input fields to send… ' + permalink
        result = brevity.shorten(text=text, permalink=permalink,
                                 permashortlink=psl)
        self.assertEqual(expected, result)
Example #9
0
 def test_shorten(self):
     for testcase in TESTS['shorten']:
         params = dict([
             (k, testcase[k]) for k in (
                 'text', 'permalink', 'permashortlink', 'permashortcitation',
                 'target_length', 'link_length', 'format',
             )
             if k in testcase])
         result = brevity.shorten(**params)
         expected = testcase['expected']
         self.assertEqual(expected, result)
Example #10
0
    def truncate(self, content, url, include_link, type=None, quote_url=None):
        """Shorten text content to fit within a character limit.

    Character limit and URL character length are taken from the
    TRUNCATE_TEXT_LENGTH and TRUNCATE_URL_LENGTH class constants

    Args:
      content: string
      url: string
      include_link: string
      type: string, optional: 'article', 'note', etc.
      quote_url: string URL, optional. If provided, it will be appended to the
        content, *after* truncating.

    Return: string, the possibly shortened and ellipsized text
    """
        kwargs = {}

        if quote_url:
            kwargs['target_length'] = (
                (self.TRUNCATE_TEXT_LENGTH
                 or brevity.WEIGHTS['maxWeightedTweetLength']) -
                (self.TRUNCATE_URL_LENGTH
                 or brevity.WEIGHTS['transformedURLLength']) - 1)
        elif self.TRUNCATE_TEXT_LENGTH is not None:
            kwargs['target_length'] = self.TRUNCATE_TEXT_LENGTH

        if self.TRUNCATE_URL_LENGTH is not None:
            kwargs['link_length'] = self.TRUNCATE_URL_LENGTH

        if include_link != OMIT_LINK:
            kwargs['permalink'] = url  # only include when text is truncated

        if include_link == INCLUDE_LINK:
            kwargs['permashortlink'] = url  # always include

        if type == 'article':
            kwargs['format'] = brevity.FORMAT_ARTICLE

        truncated = brevity.shorten(content, **kwargs)

        if quote_url:
            truncated += ' ' + quote_url

        return truncated
Example #11
0
  def _truncate(self, content, url, include_link, type):
    """Shorten tweet content to fit within the 140 character limit.

    Args:
      content: string
      url: string
      include_link: string
      type: string: 'article', 'note', etc.

    Return: string, the possibly shortened and ellipsized tweet text
    """
    if type == 'article':
      format = brevity.FORMAT_ARTICLE
    else:
      format = brevity.FORMAT_NOTE

    return brevity.shorten(
      content,
      # permalink is included only when the text is truncated
      permalink=url if include_link != source.OMIT_LINK else None,
      # permashortlink is always included
      permashortlink=url if include_link == source.INCLUDE_LINK else None,
      target_length=MAX_TWEET_LENGTH, link_length=TCO_LENGTH, format=format)
Example #12
0
def publish(site):
    auth = OAuth1(client_key=current_app.config['TWITTER_CLIENT_KEY'],
                  client_secret=current_app.config['TWITTER_CLIENT_SECRET'],
                  resource_owner_key=site.account.token,
                  resource_owner_secret=site.account.token_secret)

    def interpret_response(result):
        if result.status_code // 100 != 2:
            return util.wrap_silo_error_response(result)

        result_json = result.json()
        twitter_url = 'https://twitter.com/{}/status/{}'.format(
            result_json.get('user', {}).get('screen_name'),
            result_json.get('id_str'))
        return util.make_publish_success_response(twitter_url, result_json)

    def get_tweet_id(original):
        tweet_url = util.posse_post_discovery(original, TWEET_RE)
        if tweet_url:
            m = TWEET_RE.match(tweet_url)
            if m:
                return m.group(1), m.group(2)
        return None, None

    def upload_photo(photo):
        current_app.logger.debug('uploading photo, name=%s, type=%s',
                                 photo.filename, photo.content_type)
        result = requests.post(UPLOAD_MEDIA_URL,
                               files={
                                   'media': (photo.filename, photo.stream,
                                             photo.content_type),
                               },
                               auth=auth)
        if result.status_code // 100 != 2:
            return None, result
        result_data = result.json()
        current_app.logger.debug('upload result: %s', result_data)
        return result_data.get('media_id_string'), None

    def upload_video(video, default_content_type='video/mp4'):
        # chunked video upload
        chunk_files = []

        def cleanup():
            for f in chunk_files:
                os.unlink(f)

        chunk_size = 1 << 20
        total_size = 0
        while True:
            chunk = video.read(chunk_size)
            if not chunk:
                break
            total_size += len(chunk)

            tempfd, tempfn = tempfile.mkstemp(
                '-%03d-%s' % (len(chunk_files), video.filename))
            with open(tempfn, 'wb') as f:
                f.write(chunk)
            chunk_files.append(tempfn)

        current_app.logger.debug('init upload. type=%s, length=%s',
                                 video.content_type, video.content_length)
        result = requests.post(UPLOAD_MEDIA_URL,
                               data={
                                   'command': 'INIT',
                                   'media_type': video.content_type
                                   or default_content_type,
                                   'total_bytes': total_size,
                               },
                               auth=auth)
        current_app.logger.debug('init result: %s %s', result, result.text)
        if result.status_code // 100 != 2:
            cleanup()
            return None, result
        result_data = result.json()
        media_id = result_data.get('media_id_string')
        segment_idx = 0

        for chunk_file in chunk_files:
            current_app.logger.debug('appending file: %s', chunk_file)
            result = requests.post(UPLOAD_MEDIA_URL,
                                   data={
                                       'command': 'APPEND',
                                       'media_id': media_id,
                                       'segment_index': segment_idx,
                                   },
                                   files={
                                       'media': open(chunk_file, 'rb'),
                                   },
                                   auth=auth)
            current_app.logger.debug('append result: %s %s', result,
                                     result.text)
            if result.status_code // 100 != 2:
                cleanup()
                return None, result
            segment_idx += 1

        current_app.logger.debug('finalize uploading video: %s', media_id)
        result = requests.post(UPLOAD_MEDIA_URL,
                               data={
                                   'command': 'FINALIZE',
                                   'media_id': media_id,
                               },
                               auth=auth)
        current_app.logger.debug('finalize result: %s %s', result, result.text)
        if result.status_code // 100 != 2:
            cleanup()
            return None, result
        cleanup()
        return media_id, None

    data = {}
    format = brevity.FORMAT_NOTE
    content = request.form.get('content[value]') or request.form.get('content')

    if 'name' in request.form:
        format = brevity.FORMAT_ARTICLE
        content = request.form.get('name')

    repost_ofs = util.get_possible_array_value(request.form, 'repost-of')
    for repost_of in repost_ofs:
        _, tweet_id = get_tweet_id(repost_of)
        if tweet_id:
            return interpret_response(
                requests.post(RETWEET_STATUS_URL.format(tweet_id), auth=auth))
    else:
        if repost_ofs:
            content = 'Reposted: {}'.format(repost_ofs[0])

    like_ofs = util.get_possible_array_value(request.form, 'like-of')
    for like_of in like_ofs:
        _, tweet_id = get_tweet_id(like_of)
        if tweet_id:
            return interpret_response(
                requests.post(FAVE_STATUS_URL,
                              data={'id': tweet_id},
                              auth=auth))
    else:
        if like_ofs:
            content = 'Liked: {}'.format(like_ofs[0])

    media_ids = []
    for photo in util.get_files_or_urls_as_file_storage(
            request.files, request.form, 'photo'):
        media_id, err = upload_photo(photo)
        if err:
            return util.wrap_silo_error_response(err)
        media_ids.append(media_id)

    for video in util.get_files_or_urls_as_file_storage(
            request.files, request.form, 'video'):
        media_id, err = upload_video(video)
        if err:
            return util.wrap_silo_error_response(err)
        media_ids.append(media_id)

    in_reply_tos = util.get_possible_array_value(request.form, 'in-reply-to')
    for in_reply_to in in_reply_tos:
        twitterer, tweet_id = get_tweet_id(in_reply_to)
        if tweet_id:
            data['in_reply_to_status_id'] = tweet_id
            break
    else:
        if in_reply_tos:
            content = 'Re: {}, {}'.format(in_reply_tos[0], content)

    location = request.form.get('location')
    current_app.logger.debug('received location param: %s', location)
    data['lat'], data['long'] = util.parse_geo_uri(location)

    permalink_url = request.form.get('url')
    if media_ids:
        data['media_ids'] = ','.join(media_ids)

    if content:
        data['status'] = brevity.shorten(content,
                                         permalink=permalink_url,
                                         format=format,
                                         target_length=280)

    # for in-reply-to tweets, leading @mentions will be looked up from the original Tweet, and added to the new Tweet from there.
    # https://developer.twitter.com/en/docs/tweets/post-and-engage/api-reference/post-statuses-update
    data['auto_populate_reply_metadata'] = 'true'

    data = util.trim_nulls(data)
    current_app.logger.debug('publishing with params %s', data)
    return interpret_response(
        requests.post(CREATE_STATUS_URL, data=data, auth=auth))
Example #13
0
def guess_tweet_content(post, in_reply_to):
    """Best guess effort to generate tweet content for a post; useful for
    auto-filling the share form.
    """
    preview = ''
    if post.title:
        preview += post.title

    # add location if it's a checkin
    elif post.post_type == 'checkin' and post.venue:
        preview = 'Checked in to ' + post.venue.name

    text_content = format_markdown_as_tweet(post.content)
    if text_content:
        preview += (': ' if preview else '') + text_content

    # add an in-reply-to if one isn't there already
    if in_reply_to:
        reply_match = PERMALINK_RE.match(in_reply_to)
        if reply_match:
            # get the status we're responding to
            status_response = requests.get(
                'https://api.twitter.com/1.1/statuses/show/{}.json'.format(
                    reply_match.group(2)),
                auth=get_auth())

            if status_response.status_code // 2 != 100:
                current_app.logger.warn(
                    'failed to fetch tweet %s %s while finding participants',
                    status_response, status_response.content)
                status_data = {}
            else:
                status_data = status_response.json()

            # get the list of people to respond to
            mentioned_users = []
            my_screen_name = get_authed_twitter_account().get(
                'screen_name', '')
            for user in status_data.get('entities',
                                        {}).get('user_mentions', []):
                screen_name = user.get('screen_name', '')

                if screen_name and screen_name.lower() != my_screen_name.lower(
                ):
                    mentioned_users.append(screen_name)
            mentioned_users.append(reply_match.group(1))  # the status author
            current_app.logger.debug('got mentioned users %s', mentioned_users)

            # check to see if anybody is already mentioned by the preview
            mention_match = USERMENTION_RE.findall(preview)
            for match in mention_match:
                if match[0] in mentioned_users:
                    break
            else:
                # nobody was mentioned, prepend all the names!
                for user in mentioned_users:
                    preview = prepend_twitter_name(user, preview)

    target_length = TWEET_CHAR_LENGTH

    img_url = None
    if post.post_type == 'photo' and post.attachments:
        img_url = post.attachments[0].url
        target_length -= MEDIA_CHAR_LENGTH

    preview = brevity.shorten(preview,
                              permalink=post.permalink,
                              target_length=target_length)
    return preview, img_url
Example #14
0
def guess_tweet_content(post, in_reply_to):
    """Best guess effort to generate tweet content for a post; useful for
    auto-filling the share form.
    """
    preview = ''
    if post.title:
        preview += post.title

    # add location if it's a checkin
    elif post.post_type == 'checkin' and post.venue:
        preview = 'Checked in to ' + post.venue.name

    text_content = format_markdown_as_tweet(post.content)
    if text_content:
        preview += (': ' if preview else '') + text_content

    # add an in-reply-to if one isn't there already
    if in_reply_to:
        reply_match = PERMALINK_RE.match(in_reply_to)
        if reply_match:
            # get the status we're responding to
            status_response = requests.get(
                'https://api.twitter.com/1.1/statuses/show/{}.json'.format(
                    reply_match.group(2)),
                auth=get_auth())

            if status_response.status_code // 2 != 100:
                current_app.logger.warn(
                    'failed to fetch tweet %s %s while finding participants',
                    status_response, status_response.content)
                status_data = {}
            else:
                status_data = status_response.json()

            # get the list of people to respond to
            mentioned_users = []
            my_screen_name = get_authed_twitter_account().get(
                'screen_name', '')
            for user in status_data.get('entities', {}).get('user_mentions', []):
                screen_name = user.get('screen_name', '')

                if screen_name and screen_name.lower() != my_screen_name.lower():
                    mentioned_users.append(screen_name)
            mentioned_users.append(reply_match.group(1))  # the status author
            current_app.logger.debug('got mentioned users %s', mentioned_users)

            # check to see if anybody is already mentioned by the preview
            mention_match = USERMENTION_RE.findall(preview)
            for match in mention_match:
                if match[0] in mentioned_users:
                    break
            else:
                # nobody was mentioned, prepend all the names!
                for user in mentioned_users:
                    preview = prepend_twitter_name(user, preview)

    target_length = TWEET_CHAR_LENGTH

    img_url = None
    if post.post_type == 'photo' and post.attachments:
        img_url = post.attachments[0].url
        target_length -= MEDIA_CHAR_LENGTH

    preview = brevity.shorten(preview, permalink=post.permalink,
                              target_length=target_length)
    return preview, img_url
Example #15
0
 def test_no_shorten_intl_characters(self):
     text = u"Si Hären Engel duurch all, Haus Benn dé blo, am wuel Kolrettchen Nuechtegall dén. Nun en schéi Milliounen, an wee drem d'Welt, do Ierd blénk"
     self.assertEqual(text, brevity.shorten(text=text))
Example #16
0
def publish(site):
    auth = OAuth1(
        client_key=current_app.config['TWITTER_CLIENT_KEY'],
        client_secret=current_app.config['TWITTER_CLIENT_SECRET'],
        resource_owner_key=site.account.token,
        resource_owner_secret=site.account.token_secret)

    def interpret_response(result):
        if result.status_code // 100 != 2:
            return util.wrap_silo_error_response(result)

        result_json = result.json()
        twitter_url = 'https://twitter.com/{}/status/{}'.format(
            result_json.get('user', {}).get('screen_name'),
            result_json.get('id_str'))
        return util.make_publish_success_response(twitter_url, result_json)

    def get_tweet_id(original):
        tweet_url = util.posse_post_discovery(original, TWEET_RE)
        if tweet_url:
            m = TWEET_RE.match(tweet_url)
            if m:
                return m.group(1), m.group(2)
        return None, None

    def upload_photo(photo):
        current_app.logger.debug('uploading photo, name=%s, type=%s',
                                 photo.filename, photo.content_type)
        result = requests.post(UPLOAD_MEDIA_URL, files={
            'media': (photo.filename, photo.stream, photo.content_type),
        }, auth=auth)
        if result.status_code // 100 != 2:
            return None, result
        result_data = result.json()
        current_app.logger.debug('upload result: %s', result_data)
        return result_data.get('media_id_string'), None

    def upload_video(video, default_content_type='video/mp4'):
        # chunked video upload
        chunk_files = []

        def cleanup():
            for f in chunk_files:
                os.unlink(f)

        chunk_size = 1 << 20
        total_size = 0
        while True:
            chunk = video.read(chunk_size)
            if not chunk:
                break
            total_size += len(chunk)

            tempfd, tempfn = tempfile.mkstemp('-%03d-%s' % (
                len(chunk_files), video.filename))
            with open(tempfn, 'wb') as f:
                f.write(chunk)
            chunk_files.append(tempfn)

        current_app.logger.debug('init upload. type=%s, length=%s',
                                 video.content_type, video.content_length)
        result = requests.post(UPLOAD_MEDIA_URL, data={
            'command': 'INIT',
            'media_type': video.content_type or default_content_type,
            'total_bytes': total_size,
        }, auth=auth)
        current_app.logger.debug('init result: %s %s', result, result.text)
        if result.status_code // 100 != 2:
            cleanup()
            return None, result
        result_data = result.json()
        media_id = result_data.get('media_id_string')
        segment_idx = 0

        for chunk_file in chunk_files:
            current_app.logger.debug('appending file: %s', chunk_file)
            result = requests.post(UPLOAD_MEDIA_URL, data={
                'command': 'APPEND',
                'media_id': media_id,
                'segment_index': segment_idx,
            }, files={
                'media': open(chunk_file, 'rb'),
            }, auth=auth)
            current_app.logger.debug(
                'append result: %s %s', result, result.text)
            if result.status_code // 100 != 2:
                cleanup()
                return None, result
            segment_idx += 1

        current_app.logger.debug('finalize uploading video: %s', media_id)
        result = requests.post(UPLOAD_MEDIA_URL, data={
            'command': 'FINALIZE',
            'media_id': media_id,
        }, auth=auth)
        current_app.logger.debug('finalize result: %s %s', result, result.text)
        if result.status_code // 100 != 2:
            cleanup()
            return None, result
        cleanup()
        return media_id, None

    data = {}
    format = brevity.FORMAT_NOTE
    content = request.form.get('content[value]') or request.form.get('content')

    if 'name' in request.form:
        format = brevity.FORMAT_ARTICLE
        content = request.form.get('name')

    repost_ofs = util.get_possible_array_value(request.form, 'repost-of')
    for repost_of in repost_ofs:
        _, tweet_id = get_tweet_id(repost_of)
        if tweet_id:
            return interpret_response(
                requests.post(RETWEET_STATUS_URL.format(tweet_id), auth=auth))
    else:
        if repost_ofs:
            content = 'Reposted: {}'.format(repost_ofs[0])

    like_ofs = util.get_possible_array_value(request.form, 'like-of')
    for like_of in like_ofs:
        _, tweet_id = get_tweet_id(like_of)
        if tweet_id:
            return interpret_response(
                requests.post(FAVE_STATUS_URL, data={'id': tweet_id}, auth=auth))
    else:
        if like_ofs:
            content = 'Liked: {}'.format(like_ofs[0])

    media_ids = []
    for photo in util.get_files_or_urls_as_file_storage(request.files, request.form, 'photo'):
        media_id, err = upload_photo(photo)
        if err:
            return util.wrap_silo_error_response(err)
        media_ids.append(media_id)

    for video in util.get_files_or_urls_as_file_storage(request.files, request.form, 'video'):
        media_id, err = upload_video(video)
        if err:
            return util.wrap_silo_error_response(err)
        media_ids.append(media_id)

    in_reply_tos = util.get_possible_array_value(request.form, 'in-reply-to')
    for in_reply_to in in_reply_tos:
        twitterer, tweet_id = get_tweet_id(in_reply_to)
        if tweet_id:
            data['in_reply_to_status_id'] = tweet_id
            break
    else:
        if in_reply_tos:
            content = 'Re: {}, {}'.format(in_reply_tos[0], content)

    location = request.form.get('location')
    current_app.logger.debug('received location param: %s', location)
    data['lat'], data['long'] = util.parse_geo_uri(location)

    permalink_url = request.form.get('url')
    if media_ids:
        data['media_ids'] = ','.join(media_ids)

    if content:
        data['status'] = brevity.shorten(content, permalink=permalink_url,
                                         format=format, target_length=280)

    # for in-reply-to tweets, leading @mentions will be looked up from the original Tweet, and added to the new Tweet from there.
    # https://developer.twitter.com/en/docs/tweets/post-and-engage/api-reference/post-statuses-update
    data['auto_populate_reply_metadata'] = 'true'

    data = util.trim_nulls(data)
    current_app.logger.debug('publishing with params %s', data)
    return interpret_response(
        requests.post(CREATE_STATUS_URL, data=data, auth=auth))