def remove_blacklisted_hosts(hosts, blacklist): to_delete = set() for host in hosts: if is_blacklisted(host, blacklist): to_delete.add(host) for host in to_delete: del hosts[host]
def get(self, ipath): if (is_blacklisted(self.request)): return self.return_error() if (len(ipath) > 11 or ipath[:6] != '/da1x/'): return self.return_error() try: parent_id = int(ipath[6:]) except ValueError: return self.return_error() path = CACHE_PATH_PREFIX + ipath expiry_boundary = datetime.now() - timedelta(seconds=CACHE_TIMEOUT) # Memcache memcache_client = memcache.Client() memcache_entry = memcache_client.get(path) if memcache_entry is not None: self.response.headers['X-Cache'] = 'HIT-M' return self.return_response(memcache_entry) # Check in DB if USE_DB: cache_entries = Cache.query(Cache.path == path).fetch(1) cache_entry = None if (len(cache_entries) == 1) and (cache_entries[0].date >= expiry_boundary): cache_entry = cache_entries[0] self.response.headers['X-Cache'] = 'HIT-D' self.return_response(cache_entries[0].content) memcache_client.set(path, cache_entry.content, (cache_entries[0].date + timedelta(seconds=CACHE_TIMEOUT) - datetime.now()).seconds) return # Go fetch! url = URL_BASE + "%d.json" % parent_id result = urlfetch.fetch(url) if result.status_code == 200: self.response.headers['Content-Type'] = 'application/json' self.response.headers['Access-Control-Allow-Origin'] = '*' self.response.out.write(result.content) if USE_DB: # Save to DB if not cache_entry: cache_entry = Cache(path=path) cache_entry.content = result.content cache_entry.put() memcache_client.set(path, result.content, CACHE_TIMEOUT) else: return self.return_error()
def get(self, path): if (is_blacklisted(self.request)): return self.return_error(); if (len(path)>20) or (path[:14] != '/api/children/' and path[:9] != '/api/tps/'): return self.return_error() expiry_boundary = datetime.now() - timedelta(seconds=CACHE_TIMEOUT) # Memcache memcache_client = memcache.Client(); memcache_entry = memcache_client.get(path) if memcache_entry is not None: self.response.headers['X-Cache'] = 'HIT-M' return self.return_response(memcache_entry) # Check in DB if USE_DB: cache_entries = Cache.query(Cache.path==path).fetch(1) cache_entry = None if (len(cache_entries) == 1) and (cache_entries[0].date >= expiry_boundary): cache_entry = cache_entries[0] self.response.headers['X-Cache'] = 'HIT-D' self.return_response(cache_entries[0].content) memcache_client.set(path, cache_entry.content, (cache_entries[0].date + timedelta(seconds=CACHE_TIMEOUT) - datetime.now()).seconds) return; # Go fetch! url = URL_BASE + path result = urlfetch.fetch(url) if result.status_code == 200: self.response.headers['Content-Type'] = 'application/json' self.response.headers['Access-Control-Allow-Origin'] = '*' self.response.out.write(result.content) if USE_DB: # Save to DB if not cache_entry: cache_entry = Cache(path=path) cache_entry.content = result.content cache_entry.put() memcache_client.set(path, result.content, CACHE_TIMEOUT) else: return self.return_error()
def short(url, custom_string=None): """ Perform the shortening of given url. Optionally use a custom string instead of using internal algorithm provided by core module Params url - the url to shorten: url custom_string - the string to use to short the link: string Return shorted url: url string """ domain_name = parse_domain(url) if is_blacklisted(domain_name): raise DomainBlacklistedError(domain_name) if custom_string and shorting_frag_already_exists(custom_string, False): raise URLExistsError(custom_string) domain = get_or_create_domain(domain_name) urlbox = None if custom_string: urlbox = UrlBox() urlbox.shorting_frag = custom_string urlbox.custom = True else: urlbox = respawn_urlbox(domain) if not urlbox: urlbox = UrlBox() short = compute_next(domain.last_shorting_frag_assigned) while shorting_frag_already_exists(domain, short): short = compute_next(short) urlbox.shorting_frag = short urlbox.custom = False urlbox.domain = domain urlbox.url = url urlbox.put() return urlbox.shorted_url