def remove_blacklisted_hosts(hosts, blacklist):
to_delete = set()
for host in hosts:
if is_blacklisted(host, blacklist):
to_delete.add(host)
for host in to_delete:
del hosts[host]
def get(self, ipath):
if (is_blacklisted(self.request)):
return self.return_error()
if (len(ipath) > 11 or ipath[:6] != '/da1x/'):
return self.return_error()
try:
parent_id = int(ipath[6:])
except ValueError:
return self.return_error()
path = CACHE_PATH_PREFIX + ipath
expiry_boundary = datetime.now() - timedelta(seconds=CACHE_TIMEOUT)
# Memcache
memcache_client = memcache.Client()
memcache_entry = memcache_client.get(path)
if memcache_entry is not None:
self.response.headers['X-Cache'] = 'HIT-M'
return self.return_response(memcache_entry)
# Check in DB
if USE_DB:
cache_entries = Cache.query(Cache.path == path).fetch(1)
cache_entry = None
if (len(cache_entries)
== 1) and (cache_entries[0].date >= expiry_boundary):
cache_entry = cache_entries[0]
self.response.headers['X-Cache'] = 'HIT-D'
self.return_response(cache_entries[0].content)
memcache_client.set(path, cache_entry.content,
(cache_entries[0].date +
timedelta(seconds=CACHE_TIMEOUT) -
datetime.now()).seconds)
return
# Go fetch!
url = URL_BASE + "%d.json" % parent_id
result = urlfetch.fetch(url)
if result.status_code == 200:
self.response.headers['Content-Type'] = 'application/json'
self.response.headers['Access-Control-Allow-Origin'] = '*'
self.response.out.write(result.content)
if USE_DB:
# Save to DB
if not cache_entry:
cache_entry = Cache(path=path)
cache_entry.content = result.content
cache_entry.put()
memcache_client.set(path, result.content, CACHE_TIMEOUT)
else:
return self.return_error()
def get(self, path):
if (is_blacklisted(self.request)):
return self.return_error();
if (len(path)>20) or (path[:14] != '/api/children/' and path[:9] != '/api/tps/'):
return self.return_error()
expiry_boundary = datetime.now() - timedelta(seconds=CACHE_TIMEOUT)
# Memcache
memcache_client = memcache.Client();
memcache_entry = memcache_client.get(path)
if memcache_entry is not None:
self.response.headers['X-Cache'] = 'HIT-M'
return self.return_response(memcache_entry)
# Check in DB
if USE_DB:
cache_entries = Cache.query(Cache.path==path).fetch(1)
cache_entry = None
if (len(cache_entries) == 1) and (cache_entries[0].date >= expiry_boundary):
cache_entry = cache_entries[0]
self.response.headers['X-Cache'] = 'HIT-D'
self.return_response(cache_entries[0].content)
memcache_client.set(path, cache_entry.content,
(cache_entries[0].date + timedelta(seconds=CACHE_TIMEOUT) - datetime.now()).seconds)
return;
# Go fetch!
url = URL_BASE + path
result = urlfetch.fetch(url)
if result.status_code == 200:
self.response.headers['Content-Type'] = 'application/json'
self.response.headers['Access-Control-Allow-Origin'] = '*'
self.response.out.write(result.content)
if USE_DB:
# Save to DB
if not cache_entry:
cache_entry = Cache(path=path)
cache_entry.content = result.content
cache_entry.put()
memcache_client.set(path, result.content, CACHE_TIMEOUT)
else:
return self.return_error()
def short(url, custom_string=None):
"""
Perform the shortening of given url. Optionally use a custom string
instead of using internal algorithm provided by core module
Params
url - the url to shorten: url
custom_string - the string to use to short the link: string
Return
shorted url: url string
"""
domain_name = parse_domain(url)
if is_blacklisted(domain_name):
raise DomainBlacklistedError(domain_name)
if custom_string and shorting_frag_already_exists(custom_string, False):
raise URLExistsError(custom_string)
domain = get_or_create_domain(domain_name)
urlbox = None
if custom_string:
urlbox = UrlBox()
urlbox.shorting_frag = custom_string
urlbox.custom = True
else:
urlbox = respawn_urlbox(domain)
if not urlbox:
urlbox = UrlBox()
short = compute_next(domain.last_shorting_frag_assigned)
while shorting_frag_already_exists(domain, short):
short = compute_next(short)
urlbox.shorting_frag = short
urlbox.custom = False
urlbox.domain = domain
urlbox.url = url
urlbox.put()
return urlbox.shorted_url
