def get_and_cache_owner(self):
cache_key = 'TickUser(%s)' % self.get_owner_key()
owner = cache.get(cache_key)
if not owner:
owner = self.owner
cache.set(cache_key,owner)
return owner
def test_cache_set_expire(mock_cache):
redis, clock = mock_cache
cache.set("foo", "bar", expires=60)
assert cache.get("foo") == "bar"
clock.set_time(datetime.datetime.now() + datetime.timedelta(seconds=61))
redis.do_expire()
assert cache.get("foo") == None
def get_pins(filters):
"""
Get pins based on filters
"""
key = get_pins_cache_key(filters)
# try getting pins from cache
pins = cache.get(key)
if pins is None:
if Server.DEBUG:
print('\033[93m' + "Pin request cache miss" + '\033[0m')
# try getting pins from DB
pins = requests.standard_query([
'srnumber',
'requesttype',
'latitude',
'longitude',
],
filters,
table='map')
# add result to cache
cache.set(key, pins)
else:
if Server.DEBUG:
print('\033[92m' + "Pin request cache hit" + '\033[0m')
return pins
def variable_init(conf_={}):
"""
Load the persistent variable table.
The variable table is composed of values that have been saved in the table
with variable_set() as well as those explicitly specified
in the configuration file.
"""
# NOTE: caching the variables improves performance by 20% when serving
# cached pages.
cached = lib_cache.get('variables', 'cache')
if (cached):
variables = cached.data
else:
variables = {}
result = lib_database.query('SELECT * FROM {variable}')
while True:
variable = lib_database.fetch_object(result)
if (not variable):
break
variables[variable.name] = php.unserialize(variable.value)
lib_cache.set('variables', variables)
for name, value in conf_.items():
variables[name] = value
return variables
def load_data():
dirname = os.path.dirname(os.path.realpath(__file__))
expedia_str = get('expedia_df')
if expedia_str is None:
filename = dirname + '/data/train.csv'
df_chunk = pd.read_csv(filename,
chunksize=10**6,
usecols=[
'prop_id', 'srch_room_count',
'visitor_location_country_id',
'srch_booking_window',
'srch_saturday_night_bool', 'price_usd',
'date_time'
],
dtype={
"prop_id": "int32",
"srch_room_count": "int32",
"visitor_location_country_id": "int32",
"srch_booking_window": "int32",
"srch_saturday_night_bool": "int32",
"price_usd": "float32",
"date_time": "object"
})
chunks = []
for chunk in df_chunk:
chunks.append(chunk)
expedia = pd.concat(chunks)
set('expedia_df', pickle.dumps(expedia))
else:
expedia = pickle.loads(expedia_str)
return expedia
def process_comment(data,**kwargs):
logging.info("$$$$ nlp.process_comment[]")
cache = voxpop.VPE.get_caches().get(key=data['cache_id'])
try:
comment = cache.get_cached()[data['comment_id']]
except KeyError:
logging.info("$$$$ nlp.process_comment: ITEM NOT FOUND IN MEMORY ["+data['comment_id']+"]")
return False
tokens = nltk.word_tokenize(nltk.clean_html(comment['commentBody']))
words = {'positive':[], 'negative':[], 'strong':[], 'weak':[], 'active':[], 'passive':[]}
for token in tokens:
with lasswell_lock:
parsed = lp.parse(token)
if parsed:
logging.info(token + str(parsed))
if parsed[0] == 1:
words['positive'].append(token)
elif parsed[0] == -1:
words['negative'].append(token)
if parsed[1] == 1:
words['strong'].append(token)
elif parsed[1] == -1:
words['weak'].append(token)
if parsed[2] == 1:
words['active'].append(token)
elif parsed[2] == -1:
words['passive'].append(token)
comment['words'] = words
cache.set(data['comment_id'], comment)
return True
def test_cache_set_expire(mock_cache):
redis, clock = mock_cache
cache.set("foo", "bar", expires=60)
assert cache.get("foo") == "bar"
clock.set_time(datetime.datetime.now() + datetime.timedelta(seconds=61))
redis.do_expire()
assert cache.get("foo") == None
def ReplicateFile(self, request, context):
print("request", request.shortest_path)
# next_node = request.shortest_path[request.currentpos]
if request.currentpos == len(request.shortest_path) - 1:
cache.set(request, request)
return fileService_pb2.ack(success=True,
message="Data Replicated.")
else:
# forward_server_addr = self.getneighbordata(next_node)
forward_server_addr = "169.105.246.6"
forward_port = 50051
forward_channel = grpc.insecure_channel(forward_server_addr + ":" +
str(forward_port))
forward_stub = fileService_pb2_grpc.FileserviceStub(
forward_channel)
request.currentpos += 1
rList = [1, 2, 3, 4, 5]
arr = bytearray(rList)
updated_request = fileService_pb2.FileData(
initialReplicaServer=request.initialReplicaServer,
bytearray=request.bytearray,
vClock=request.vClock,
shortest_path=request.shortest_path,
currentpos=request.currentpos + 1)
forward_resp = forward_stub.ReplicateFile(updated_request)
print("forward_resp", forward_resp)
return fileService_pb2.ack(success=True, message="Data Forwarded.")
def variable_init(conf_={}):
"""
Load the persistent variable table.
The variable table is composed of values that have been saved in the table
with variable_set() as well as those explicitly specified
in the configuration file.
"""
# NOTE: caching the variables improves performance by 20% when serving
# cached pages.
cached = lib_cache.get("variables", "cache")
if cached:
variables = cached.data
else:
variables = {}
result = lib_database.query("SELECT * FROM {variable}")
while True:
variable = lib_database.fetch_object(result)
if not variable:
break
variables[variable.name] = php.unserialize(variable.value)
lib_cache.set("variables", variables)
for name, value in conf_.items():
variables[name] = value
return variables
def inject_global_snippet(context, content):
if not valid_view(context):
return
from E-Commerce.xtheme import get_current_theme
from E-Commerce.xtheme.models import Snippet, SnippetType
shop = get_shop(context["request"])
cache_key = GLOBAL_SNIPPETS_CACHE_KEY.format(shop_id=shop.id)
snippets = cache.get(cache_key)
if snippets is None:
snippets = Snippet.objects.filter(shop=shop)
cache.set(cache_key, snippets)
for snippet in snippets:
if snippet.themes:
current_theme = get_current_theme(shop)
if current_theme and current_theme.identifier not in snippet.themes:
continue
content = snippet.snippet
if snippet.snippet_type == SnippetType.InlineJS:
content = InlineScriptResource(content)
elif snippet.snippet_type == SnippetType.InlineCSS:
content = InlineStyleResource(content)
elif snippet.snippet_type == SnippetType.InlineHTMLMarkup:
content = InlineMarkupResource(content)
add_resource(context, snippet.location, content)
def thumbnail(source, alias=None, generate=True, **kwargs):
if not source:
return None
cache_key, cached_thumbnail_url = _get_cached_thumbnail_url(source, alias=alias, generate=generate, **kwargs)
if cached_thumbnail_url is not None:
return cached_thumbnail_url
thumbnailer_instance = get_thumbnailer(source)
if not thumbnailer_instance:
return None
if _is_svg(thumbnailer_instance):
return source.url if hasattr(source, 'url') else None
if alias:
options = aliases.get(alias, target=thumbnailer_instance.alias_target)
options.update(process_thumbnailer_options(kwargs))
else:
options = process_thumbnailer_options(kwargs)
try:
thumbnail_instance = thumbnailer_instance.get_thumbnail(options, generate=generate)
thumbnail_url = thumbnail_instance.url
if cache_key:
cache.set(cache_key, thumbnail_url)
return thumbnail_url
except (IOError, InvalidImageFormatError, ValueError):
return None
def test_get_multi_prefix(mock_cache):
redis, clock = mock_cache
cache.set("test-foo1", "bar1")
cache.set("test-foo2", "bar2")
assert cache.get_multi(["foo1", "foo2"], key_prefix="test-") == {
"foo1": "bar1",
"foo2": "bar2"
}
def test_get_multi(mock_cache):
redis, clock = mock_cache
cache.set("foo1", "bar1")
cache.set("foo2", "bar2")
assert cache.get_multi(["foo1", "foo2"]) == {
"foo1": "bar1",
"foo2": "bar2"
}
def load_abi(address):
key = "abi:{}".format(address)
abi = cache.get(key)
if abi is None:
res = requests.get("http://api.etherscan.io/api" + "?module=contract" +
"&action=getabi" + "&address={}".format(address))
abi = res.json().get("result", None)
cache.set(key, abi)
return abi
def get_cached_children(self):
from E-Commerce.core import cache
key = "category_cached_children:{}".format(self.pk)
children = cache.get(key)
if children is not None:
return children
children = self.get_children()
cache.set(key, children)
return children
def putNonUserChange(self):
"""Saves a non-user change to a model object and updates the cache"""
# Note: This calls db.Model, not super() to avoid an infinite recursion
# Because of this, Cached MUST be specified right before db.Model in the ancestor class list
db.Model.put(self)
cacheKey = "%s.%s" % (self.__class__, self.key())
cache.set(cacheKey, self)
def set_cached_value(key, value, timeout=None):
"""
Set value to context cache
:param key: Unique key formed to the context
:param value: Value to cache
:param timeout: Timeout as seconds
:type timeout: int
"""
cache.set(key, value, timeout=timeout)
def run(self, shop):
cache_key = self.cache_key_fmt % (self.variable_values["supplier"].pk, self.variable_values["product"].pk)
# do not run this if the last dispatch was < 1 minute
last_dispatch_time = cache.get(cache_key)
if last_dispatch_time and time() - last_dispatch_time < 60:
return
cache.set(cache_key, time(), timeout=(60 * 60 * 24))
super(AlertLimitReached, self).run(shop=shop)
def poll_opensecrets(state: str, cache_only: bool = True):
try:
state = state.upper()
if cache.get(f"opensecrets_{state}") is None:
if cache_only:
print(f"Loading state {state} in the background...")
thread = threading.Thread(
target=lambda: poll_opensecrets(state, cache_only=False))
thread.setDaemon(True)
thread.start()
return {}
people_dict = {}
ids = requests.get(
"http://www.opensecrets.org/api/",
params={
"method": "getLegislators",
"id": state,
"output": "json",
"apikey": random.choice(OPENSECRETS_API_KEYS),
},
).json()["response"]
if not isinstance(ids["legislator"], list):
# Handle cases where there is only one legislator and the API compacts the response
ids["legislator"] = [ids["legislator"]]
for legislator in ids["legislator"]:
try:
attributes = legislator["@attributes"]
cid = attributes["cid"]
name = attributes["firstlast"]
name_dict = {"name": name, "cid": cid}
secrets = requests.get(
"http://www.opensecrets.org/api/",
params={
"method": "candSummary",
"cid": cid,
"output": "json",
"apikey": random.choice(OPENSECRETS_API_KEYS),
},
).json()["response"]["summary"]["@attributes"]
name_dict["secrets"] = secrets
people_dict[name] = name_dict
except Exception as e:
print(
f"an error occured when trying to load {legislator}: {e}"
)
print(f"Loaded opensecrets data for {state}!")
cache.set(f"opensecrets_{state}", people_dict)
return cache.get(f"opensecrets_{state}")
except Exception as e:
print(f"could not poll OpenSecrets: {e}")
return {}
def _get_resource(self, request, resource_id):
cache_key = "E-CommerceCOM_API_%s_%s" % (request.LANGUAGE_CODE, resource_id)
resource = cache.get(cache_key)
if not resource:
try:
r = requests.get("https://www.E-Commerce.com/%s/api/%s/" % (request.LANGUAGE_CODE, resource_id))
resource = r.json()
cache.set(cache_key, resource, timeout=SECONDS_IN_DAY)
except Exception:
pass
return resource or {}
def get_contract_translator(address):
key = "ct:{}".format(address)
if cache.has_key(key):
ct = cache.get(key)
else:
abi = load_abi(address)
if (abi is not None) and (len(abi) > 1):
ct = ethereum.abi.ContractTranslator(abi)
else:
ct = None
cache.set(key, ct)
return ct
def phid_to_name(phid):
if phid is None:
return None
if cache.has(phid):
return cache.get(phid)
phab = Phabricator()
data = phab.phid.lookup(names=[
phid,
])
name = data[phid]['name']
cache.set(phid, name)
return name
def get_pins(filters):
key = pins_key(filters)
pins = cache.get(key)
if pins is None:
pins = requests.standard_query(
['srnumber', 'requesttype', 'latitude', 'longitude'],
filters,
table='map')
cache.set(key, pins)
return pins
def get_matching_context_conditions(context):
namespace = CONTEXT_CONDITION_CACHE_NAMESPACE
ctx_cache_elements = dict(
customer=context.customer.pk or 0,
shop=context.shop.pk)
conditions_cache_key = "%s:%s" % (namespace, hash(frozenset(ctx_cache_elements.items())))
matching_context_conditions = cache.get(conditions_cache_key, None)
if matching_context_conditions is None:
matching_context_conditions = set()
for condition in ContextCondition.objects.filter(active=True):
if condition.matches(context):
matching_context_conditions.add(condition.pk)
cache.set(conditions_cache_key, matching_context_conditions, timeout=None)
return matching_context_conditions
def get_email_template(name):
"""
Function to get email template object that checks from cache first if caching is enabled
"""
if hasattr(settings, 'POST_OFFICE_CACHE') and settings.POST_OFFICE_TEMPLATE_CACHE is False:
return EmailTemplate.objects.get(name=name)
else:
email_template = cache.get(name)
if email_template is not None:
return email_template
else:
email_template = EmailTemplate.objects.get(name=name)
cache.set(name, email_template)
return email_template
def get_data_from_page_or_cache(user, region, platform):
data = find_user(user, region, platform)
if not data:
return None
if data[1]:
stats = data[1]
else:
page, region, battletag = data[0]
stats = parsers.parse_stats(page, region, battletag, platform)
cache.set(user + region + platform, stats, 1200)
return stats
def getData():
data = ""
logging.debug("is_caching_enabled:" + caching_enabled)
if caching_enabled == "true":
data = cache.get()
if data == "":
providerData = mongo.getData()
bloodData = postgres.getData()
data = combineData(providerData, bloodData)
if caching_enabled == "true":
cache.set(data)
return data
def getFromProperty(cls, prop, value):
clsname = cls.__name__
key = "%s.%s=%s" % (clsname, prop, value)
result = cache.get(key)
if result:
return result
q = db.GqlQuery("SELECT * FROM %s WHERE %s = :v" % (clsname, prop), v=value)
objects = q.fetch(1)
if len(objects) >= 1:
result = objects[0]
cache.set(key, result)
return result
cache.set(key, result)
return None
def fetch(url, time=0):
"""Fetch a URL, caching the result"""
result = cache.get(url, namespace="kiwi.f")
if result:
return result
try:
fetchResult = urlfetch.fetch(url)
if fetchResult.status_code == 200:
result = fetchResult.content
cache.set(url, result, time=time, namespace="kiwi.f")
return result
except:
pass
return ""
def test_cache_api():
with override_settings(CACHES={
'default': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'LOCATION': 'test_configuration_cache',
}
}):
cache.init_cache()
key = "test_prefix:123"
value = "456"
cache.set(key, value)
assert cache.get(key) == value
cache.bump_version(key)
assert cache.get(key, default="derp") == "derp" # version was bumped, so no way this is there
cache.set(key, value)
assert cache.get(key) == value
def get_matching(cls, context, shop_product):
prod_ctx_cache_elements = dict(
customer=context.customer.pk or 0,
shop=context.shop.pk,
product_id=shop_product.pk)
namespace = CAMPAIGNS_CACHE_NAMESPACE
key = "%s:%s" % (namespace, hash(frozenset(prod_ctx_cache_elements.items())))
cached_matching = cache.get(key, None)
if cached_matching is not None:
return cached_matching
from E-Commerce.campaigns.models.matching import get_matching_context_conditions, get_matching_catalog_filters
matching_context_conditions = get_matching_context_conditions(context)
matching_catalog_filters = get_matching_catalog_filters(shop_product)
if not (matching_context_conditions or matching_catalog_filters):
return []
# Get all possible campaign id's for matching context_conditions
campaigns_based_on_conditions = set(
cls.objects.filter(
active=True,
shop=context.shop,
conditions__id__in=matching_context_conditions
).values_list("pk", flat=True)
)
campaigns_based_on_catalog_filters = set()
if hasattr(cls, "filters"):
# Get all possible campaigns for matching catalog_filters
campaigns_based_on_catalog_filters = set(
cls.objects.filter(
active=True,
shop=context.shop,
filters__id__in=matching_catalog_filters
).values_list("pk", flat=True)
)
all_possible_campaigns_ids = (campaigns_based_on_conditions | campaigns_based_on_catalog_filters)
matching = []
for campaign in cls.objects.filter(id__in=all_possible_campaigns_ids):
if campaign.rules_match(context, shop_product, matching_catalog_filters, matching_context_conditions):
matching.append(campaign)
cache.set(key, matching, timeout=None)
return matching
def scrape_page(url):
"""
Scrapes a web page given a url and returns an HTML tree representation.
This will check for a cachced version of the page before scraping and will attempt to cache
after scraping
:param url: A string of the url of the requested web page
:return: html tree structure based on the html markup of the scraped website
"""
cached_page = cache.get(url)
if cached_page:
return html.fromstring(cached_page)
else:
page = get(url)
cache.set(url, page.text)
return html.fromstring(page.text)
def batch_set_resty_upstream(group, upstream, oper):
if group == None or upstream == None: return
if oper not in ['up', 'down']: return
all_openresty = GroupModel().allOpenrestys(group['id'])
if len(all_openresty) == 0:
log.error('group:%s has no openresty', group['name'])
return
from gevent import monkey
monkey.patch_socket()
import gevent
greenlets = [
gevent.spawn(set_resty_upstream, openresty, upstream, oper)
for openresty in all_openresty
]
gevent.joinall(greenlets)
key = cache.gen_upstream_key(upstream['host'], upstream['port'])
cache.set(key, upstream['status'])
log.info('set cache finish')
def _cache_shop_configuration(shop):
"""
Cache global or shop specific configuration.
Global configuration (`shop` is ``None``) is read first, then `shop`
based configuration is updated over that.
:param shop: Shop to cache configuration for, or None
:type shop: E-Commerce.core.models.Shop|None
:return: Cached configuration
:rtype: dict
"""
configuration = {}
configuration.update(_get_configuration_from_db(None))
if shop:
configuration.update(_get_configuration_from_db(shop))
cache.set(_get_cache_key(shop), configuration)
return configuration
def scrape_page(url):
"""
Scrapes a web page given a url and returns an HTML tree representation.
This will check for a cachced version of the page before scraping and will attempt to cache
after scraping
:param url: A string of the url of the requested web page
:return: html tree structure based on the html markup of the scraped website
"""
cached_page = cache.get(url)
if cached_page:
return html.fromstring(cached_page)
else:
page = get(url)
cache.set(url, page.text)
return html.fromstring(page.text)
def set_current_theme(identifier, shop):
"""
Activate a theme based on identifier.
:param identifier: Theme identifier
:type identifier: str
:param shop: Shop to fetch the theme settings
:type shop: E-Commerce.core.models.Shop
:return: Activated theme
:rtype: Theme
"""
cache.bump_version(get_theme_cache_key(shop))
theme = get_theme_by_identifier(identifier, shop)
if not theme:
raise ValueError("Invalid theme identifier")
theme.set_current()
cache.set(get_theme_cache_key(shop), theme)
set_middleware_current_theme(theme)
return theme
def email_captcha():
# /email_captcha/[email protected]
email = request.args.get('email')
if not email:
return restful.params_error('请传递邮箱参数！')
source = list(string.ascii_letters)
source.extend(map(lambda x: str(x), range(0, 10)))
# source.extend(["0","1","2","3","4","5","6","7","8","9"])
captcha = "".join(random.sample(source, 6))
# 给这个邮箱发送邮件
message = Message('邮箱验证码', recipients=[email], body='您的验证码是：%s' % captcha)
try:
mail.send(message)
except:
return restful.server_error()
cache.set(email, captcha)
return restful.success()
def get_path_to_top_task(self):
cache_key = '%s.get_path_to_top_task(%s,%s)'%(self.__class__.__name__,self.key().id(),self.version)
path = cache.get(cache_key)
if not path:
candidates = TickTask.all().ancestor(self).filter('prev_sibling =',None).filter('deleted =',False).fetch(1000)
if candidates:
mapping = {}
for candidate in candidates:
mapping[candidate.id] = candidate
if candidate.parent_task == None:
parent = candidate
current = parent
path = [current]
while current.first_child is not None:
current = mapping[current.first_child]
path.append(current)
else:
path = []
cache.set(cache_key,path)
return path
def get_best_selling_product_info(shop_ids, cutoff_days=30):
shop_ids = sorted(map(int, shop_ids))
cutoff_date = datetime.date.today() - datetime.timedelta(days=cutoff_days)
cache_key = "best_sellers:%r_%s" % (shop_ids, cutoff_date)
sales_data = cache.get(cache_key)
if sales_data is None:
sales_data = (
OrderLine.objects
.filter(
order__shop_id__in=shop_ids,
order__order_date__gte=to_aware(cutoff_date),
type=OrderLineType.PRODUCT
)
.values("product")
.annotate(n=Sum("quantity"))
.order_by("-n")[:100]
.values_list("product", "product__variation_parent_id", "n")
)
cache.set(cache_key, sales_data, 3 * 60 * 60) # three hours
return sales_data
def get (self):
articles = Node.get_all_articles ()
self.values['articles'] = articles
self.values['urls'] = Node.get_urls ()
site_domain = self.values.get ('site_domain', None)
site_domain_sync = Datum.get ('site_domain_sync', None)
mentions_web = cache.get ('mentions_web')
if mentions_web == None and site_domain:
link = ''.join (['http://blogsearch.google.com/blogsearch_feeds?hl=en&q=',
urllib.quote('link:' + site_domain),
'&ie=utf-8&num=10&output=atom'])
mentions_web = feedparser.parse (link)
cache.set ('mentions_web', mentions_web, 600)
if mentions_web :
self.values ['mentions_web'] = mentions_web.entries
else:
self.values ['mentions_web'] = None
mentions_twitter = cache.get('mentions_twitter')
if mentions_twitter is None:
q = None
if site_domain_sync is None:
q = site_domain
else:
q = ' OR '.join ([site_domain, site_domain_sync])
q = urllib.quote (q)
try:
result = urlfetch.fetch('http://search.twitter.com/search.json?q=' + q)
if result.status_code == 200:
mentions_twitter = simplejson.loads(result.content)
cache.add('mentions_twitter', mentions_twitter, 600)
except:
mentions_twitter = None
if mentions_twitter and len(mentions_twitter['results']) > 0:
self.values['mentions_twitter'] = mentions_twitter['results']
else:
self.values['mentions_twitter'] = None
self.render ('backstage/overview.html', **self.values)
def get_articles(area):
ret = []
for person in Site.CONTEXT.config.staff.sections():
feed = Site.CONTEXT.config.staff.get(person, config_field)
cache_key = os.path.join(
Site.CONTEXT.config.cache.get("cache","cache_dir"),
md5(feed).hexdigest())
blog = cache.get(cache_key,
expires=Site.CONTEXT.config.cache.get("cache","expires"))
if not blog:
blog = feedparser.parse(feed)
if not blog.has_key("status") or blog["status"] != 200:
print "warning:innovation:%s:feed %s return a non valid status. Skipping..." % (area, feed)
continue
cache.set(cache_key, blog)
if len(blog.feed) == 0:
print "warning:innovation:%s:feed %s is not available" % (
area, feed
)
continue
# Remove appeded title for category based feeds in wordpress
if blog.feed.title.find(u"»") != -1:
blog.feed.title = blog.feed.title.split(u"»")[0]
for e in blog.entries:
if getattr(e,"tags", False):
terms = map(lambda x:x["term"].lower(),e.tags)
if "idea" in terms or "labs" or "lab" in terms:
if "%s" % area in terms:
ret.append({"link":e.link,"title":e.title})
return ret
def test_get_multi(mock_cache):
redis, clock = mock_cache
cache.set("foo1", "bar1")
cache.set("foo2", "bar2")
assert cache.get_multi(["foo1", "foo2"]) == {"foo1": "bar1", "foo2": "bar2"}
def test_get_multi_prefix(mock_cache):
redis, clock = mock_cache
cache.set("test-foo1", "bar1")
cache.set("test-foo2", "bar2")
assert cache.get_multi(["foo1", "foo2"], key_prefix="test-") == {"foo1": "bar1", "foo2": "bar2"}
def cache(self):
cache.set(cache.make_key(self.url), {
'files': self.files,
'discovered': self.discovered,
})
def cache(self):
data = dict(self.data)
data['cache_key'] = self.cache_key
cache.set(self.cache_key, json.dumps(data))
def _set_bucket_info(self, tokens, last_accessed):
return cache.set(BUCKET_CACHE_KEY_PREFIX + self.name, "%s|%s" % (tokens, now))
feed = Site.CONTEXT.config.staff.get(person, config_fields)
cache_key = os.path.join(
Site.CONTEXT.config.cache.get("cache","cache_dir"),
md5(feed).hexdigest())
blog = cache.get(cache_key,
expires=Site.CONTEXT.config.cache.get("cache","expires"))
if not blog:
blog = feedparser.parse(feed)
if not blog.has_key("status") or int(blog["status"]) != 200:
print "warning:planet_blog:feed %s returns a non valid status" % feed
continue
cache.set(cache_key, blog)
if len(blog.feed) == 0:
print "warning:planet_blog:feed %s is not available" % feed
continue
# Remove appeded title for category based feeds in wordpress
if blog.feed.title.find(u"»") != -1:
blog.feed.title = blog.feed.title.split(u"»")[0]
Site.CONTEXT.planet.blog.feed.append(AttrDict(
title = blog.feed.title,
link = blog.feed.link,
author = unicode(person,"utf-8") ))
for e in blog.entries:
# test_cache.py
import random
import string
import cache
def random_string(length):
s = ''
for i in range(length):
s = s + random.choice(string.ascii_letters)
return s
cache.init()
for n in range(1000):
while True:
key = random_string(20)
if cache.contains(key):
continue
else:
break
value = random_string(20)
cache.set(key, value)
print("After {} iterations, cache has {} entries".format(n+1, cache.size()))
def test_cache_set(mock_cache):
redis, clock = mock_cache
cache.set("foo", "bar")
assert cache.get("foo") == "bar"
def replace_page(self, response):
response = self.ungzipResponse(response).read()
cache.set(self.markup_cache_key, response, expires = cache.TIME_HOUR * 3)
def render_template_to_response(request, templateName, responseType=HttpResponse, allowDefaultTemplates=False, allowRetry=True, args=None):
"""Renders a specified template to a string, using the specified response type
(which gets overriden for errors and automatic permanent redirects).
If allowDefaultTemplates is True, allows the default templates built into Kiwi to be found.
Returns a 404 response if the template requested does not exist or the name is illegally formed.
"""
if args == None:
args = request.args
filepath = cache.get(templateName, namespace="kiwi.t")
fromCache = False
if filepath == "":
# We cached the fact that the file didn't exist
filepath = None
elif filepath:
fromCache = True
else:
templateDir = kiwioptions.webDirectoryPath(request)
filepath = find_template(templateDir, templateName)
if not filepath and allowDefaultTemplates:
filepath = find_template(kiwioptions.KIWI_TEMPLATES_PATH, templateName)
if filepath:
cache.set(templateName, filepath, 300, namespace="kiwi.t")
else:
cache.set(templateName, "", 60, namespace="kiwi.t")
if not filepath:
return error404(request)
# If we get a HEAD request for a page that we'll be rendering automatically, accept it and render it as empty
if request.method == "HEAD":
return responseType()
try:
# BOGUS consider caching the contents of the file
f = open(filepath)
contents = f.read()
f.close()
# We differentiate Kiwi markup files by a suffix (normally .kiwi)
if filepath.endswith(kiwioptions.WEB_MARKUP_SUFFIX):
args["IS_KIWI_MARKUP"] = True
s = kiwimarkup.render(request, contents, args) # BOGUS: Cache result
return responseType(s)
# A Django template file must begin with "{". If no directive is needed, a {# Django #} comment can be used.
if contents == "" or contents[0] != "{":
return responseType(contents)
if "{~" in contents:
# Handle {~ auth ~} and {~ admin ~}
contents = contents.replace("{~ admin ~}", "")
args["IS_DJANGO"] = True
t = Template(contents) # BOGUS: Cache this instead of just the pathname
c = RequestContext(request, args)
s = t.render(c)
response = responseType(s)
return response
except Exception, ex:
if fromCache:
# Cache error can be caused by filename changing when app is updated
# Call ourselves recursively after deleting the cache entry
cache.delete(templateName, namespace="kiwi.t")
return render_template_to_response(request, templateName, responseType=responseType,
allowDefaultTemplates=allowDefaultTemplates, allowRetry=False, args=args)
else:
return error500(request)
