def bookmarklet_follow(request):
    uri = request.GET["u"]
    title = request.GET["t"]

    # url opener
    redirect_handler = HTTPRedirectHandler()
    redirect_handler.max_redirections = settings.CONFIG["url"]["max_redirections"]
    opener = urllib2.build_opener(redirect_handler)

    # web page loading
    handle = opener.open(uri)
    encoding = detect_header_encoding(handle.headers.dict)
    content = decode_html(handle.read(), encoding)
    handle.close()

    # build a resource
    discovered_resource = DiscoveredResource()
    discovered_resource.uri = uri
    discovered_resource.title = title
    discovered_resource.content = content

    # process the discovered resource
    resource = processing_service_client.process(discovered_resource)

    topic = topic_manager.create_from_features(resource.title, resource.terms, resource.entities)

    return redirect("topic_tracking_web.demo.views.topics_show", topic._id)
    def _process_web_page(self, resource):

        # if the 'http://' doesn't exist
        if (resource.uri[:7] != 'http://'):
            resource.uri = 'http://' + resource.uri
        entire_content = ''

        try:
            handle = self._opener.open(resource.uri)
            resource.uri = handle.url
            encoding = detect_header_encoding(handle.headers.dict)
            entire_content = decode_html(handle.read(), encoding)
            resource.content = entire_content
            handle.close()
            self._logger.info('Reading %s. Success.' % resource.uri)
            self._enqueue(resource)
        except (IOError, HTTPException), e:
            # mark for retry
            self._logger.error('Reading %s. IO error %s.' % (resource.uri, e))
from thrift.protocol import TBinaryProtocol
from thrift.transport import TSocket, TTransport
from topic_tracking.service.text_extraction import TextExtractionService
from topic_tracking.util.http import detect_header_encoding
from topic_tracking.util.xml import decode_html
import urllib

transport = TSocket.TSocket('localhost', 9090)
transport = TTransport.TFramedTransport(transport)
protocol = TBinaryProtocol.TBinaryProtocol(transport)

url = 'http://edition.cnn.com/2010/US/11/29/wikileaks.new.documents/index.html?hpt=T1'
handle = urllib.urlopen(url)
encoding = detect_header_encoding(handle.headers.dict)
html = decode_html(handle.read(), encoding)

client = TextExtractionService.Client(protocol)
transport.open()
content = client.extract(html)
transport.close()

print(content.__class__)
print(content)