def post_link(): link = request.values.get('q') author = request.values.get('author') key = hashlib.md5(link).hexdigest() exists = Link.get_by_key_name(key) try: if not exists: link = Link(key_name=key, link_url=link, authors=[author]) link.put() taskqueue.add(url='/_worker/fetch_title', params={ 'url': link.link_url, 'key': key }) else: if author not in exists.authors: exists.authors.append(author) exists.updated_at = datetime.now() exists.put() taskqueue.add(url='/_worker/fetch_title', params={ 'url': link, 'key': key }) return jsonify({'result': True}) except: logging.exception('api call failed') return jsonify({'result': False})
def fetch_title(): url = request.form.get("url") key = request.form.get("key") link = Link.get_by_key_name(key) if not link: return ("Model %s does not exist" % key), 404 result = urlfetch.fetch(url=url, follow_redirects=True, headers={"Accept": "text/html"}, method=urlfetch.HEAD) if result.status_code == 200 and result.headers.get("Content-Type").startswith("text/html"): result = urlfetch.fetch(url=url, follow_redirects=True, headers={"Accept": "text/html"}, method=urlfetch.GET) if result.status_code == 200: _search = _TITLE_RE_.search(result.content) if _search: r_title = _search.groups()[0] guess_encoding = chardet.detect(r_title) logging.error("title: %s guess: %s" % (r_title, repr(guess_encoding))) link.title = unicode(r_title, guess_encoding["encoding"]) link.put() return u"Link %s title updated with %s." % (url, link.title), 200 else: return u"Link %s does not have html title", 200 else: return u"Link %s return status %d" % (url, result.status_code), 500 else: return u"Link %s return status %d" % (url, result.status_code), 500