def process_data(self, controller): from google.appengine.api import memcache memcache.flush_all() obj = controller.content v = controller.form.validate_result replace_body = True for k in v: if k == 'body' and not v[k]: replace_body = False else: setattr(obj, k, v[k]) if replace_body: f = controller.request.body_file.vars["body"] v['content_type'] = detect_imagetype(v['body']) i = images.Image(v['body']) v['width'], v['height'] = i.width, i.height v['filename'] = f.filename if not v.get('name', None) and v.get('filename'): v['name'] = v['filename'] if not v.get('title', None): v['title'] = v['name'] obj.modified_at = datetime.now() obj.put() obj.sync_path(obj.get_path()) controller.set_state(FC.INITIAL) controller.redirect(obj.get_parent().get_path()+'/list')
def test_delete_db_ndb_mixed(self): # Start empty storage_ndb = StorageByKeyName( CredentialsNDBModel, 'foo', 'credentials') storage = StorageByKeyName( CredentialsModel, 'foo', 'credentials') # First DB, then NDB self.assertEqual(None, storage.get()) storage.put(self.credentials) self.assertNotEqual(None, storage.get()) storage_ndb.delete() self.assertEqual(None, storage.get()) # First NDB, then DB self.assertEqual(None, storage_ndb.get()) storage_ndb.put(self.credentials) storage.delete() self.assertNotEqual(None, storage_ndb.get()) # NDB uses memcache and an instance cache (Context) ndb.get_context().clear_cache() memcache.flush_all() self.assertEqual(None, storage_ndb.get())
def snippet_edit(request, snippet_id, profile_callback=None): snippet = Snippet.get_by_id(int(snippet_id)) if not snippet: raise Http404 if request.method == "GET": tags = "" if snippet.tags: tags = ",".join([t.name for t in snippet.get_tags() if t]) form = SnippetForm( { "title": snippet.title, "language": snippet.language, "code": snippet.code, "url": snippet.url, "comment": snippet.comment, "private": snippet.private, "tag": tags, } ) if request.method == "POST": form = SnippetForm(request.POST) logging.getLogger().debug(form) if form.is_valid(): updated_snippet = form.edit(int(snippet_id), profile_callback=profile_callback) memcache.flush_all() ping() return HttpResponseRedirect(updated_snippet.get_absolute_url()) return render_to_response( "snippet/snippet_edit.html", {"form": form, "snippet": snippet}, context_instance=RequestContext(request) )
def setup_for_testing(require_indexes=True): """Sets up the stubs for testing. Args: require_indexes: True if indexes should be required for all indexes. """ from google.appengine.api import apiproxy_stub_map from google.appengine.api import memcache from google.appengine.tools import dev_appserver from google.appengine.tools import dev_appserver_index import urlfetch_test_stub before_level = logging.getLogger().getEffectiveLevel() try: logging.getLogger().setLevel(100) root_path = os.path.realpath(os.path.dirname(__file__)) dev_appserver.SetupStubs( TEST_APP_ID, root_path=root_path, login_url='', datastore_path=tempfile.mktemp(suffix='datastore_stub'), history_path=tempfile.mktemp(suffix='datastore_history'), blobstore_path=tempfile.mktemp(suffix='blobstore_stub'), require_indexes=require_indexes, clear_datastore=False) dev_appserver_index.SetupIndexes(TEST_APP_ID, root_path) apiproxy_stub_map.apiproxy._APIProxyStubMap__stub_map['urlfetch'] = \ urlfetch_test_stub.instance # Actually need to flush, even though we've reallocated. Maybe because the # memcache stub's cache is at the module level, not the API stub? memcache.flush_all() finally: logging.getLogger().setLevel(before_level)
def get(self): date = self.request.get('date') if date == backwards_date: date = get_prev_back_date() #don't index too much if date.year < 2013 and date.month < 9: return elif date == yesterday_date: #get the yesterday's date #this is for ensuring that we indexedeverything from yesterday logging.info('today is %s' % datetime.datetime.now()) date = datetime.datetime.now() - datetime.timedelta(days=1) logging.info('indexing yesterday %s' % date) #datestr = '%s.%s.%s' % (date.day, date.month, date.year) else: try: date = datetime.datetime.strptime(date, '%d.%m.%Y') except: logging.warning('bad date in request, assuming current') date = day_date(datetime.datetime.now()) datestr = "%s.%s.%s" % (date.day, date.month, date.year) self.fetch(datestr) memcache.flush_all() keyname = str(date) models.IndexedDate.get_or_insert(keyname, date=date) self.redirect('/admin_')
def main(): site_ds = getattr(settings, 'IMPORT_SOURCE', None) logging.info(site_ds) if not site_ds: site_ds = CURRENT_SOURCE() max_groups = getattr(settings, 'MAX_GROUPS', 0) collected = [] c = 0 for (gid, gname) in site_ds.group_ids(): gdata = site_ds.group_data(gid) logging.info("%s: %s" % (gid, gdata.week(UPPER_WEEK)[0][1].list())) if not empty_data(gdata): collected.append(site_ds.group_data(gid)) if max_groups: if c < max_groups: c += 1 else: break new_v = GsqlDataSource.save_new_version(site_ds.groups(), site_ds.faculties(), site_ds.years(), collected) memcache.flush_all() print 'Fetched new version %s' % new_v
def post(self): # Create a new project. self.check_user() project = get_project_from_request(self.request) project.put() memcache.flush_all() new_story(self, "created the project", project=project) self.redirect("/projects/" + str(project.key()))
def get(self, action): if action == "flush": self.response.out.write('Flushing...') memcache.flush_all() self.redirect('/') else: self.error(404)
def post(self): self._require_admin() flushed = list() if self.request.get("all_keys") == "all_keys": memcache.flush_all() flushed.append("all memcache values") if self.request.get("webcast_keys") == "webcast_keys": flushed.append(MemcacheWebcastFlusher.flush()) if self.request.get('memcache_key') is not "": memcache.delete(self.request.get("memcache_key")) flushed.append(self.request.get("memcache_key")) if self.request.get('return_url') is not "": self.redirect("{}?flushed={}".format(self.request.get('return_url'), flushed)) return self.template_values.update({ "flushed": flushed, "memcache_stats": memcache.get_stats(), }) path = os.path.join(os.path.dirname(__file__), '../../templates/admin/memcache_index.html') self.response.out.write(template.render(path, self.template_values))
def test_get_week_numbers_with_populated_cache(self): logging.info("load get_weeks populated cache...") memcache.flush_all() self.__load_memcache_with_weeks() self.__get_weeks_test(2012,[1,2,3,4,5,6,7,8,9,10,11,12,13]) self.__get_weeks_test(2013,[1,2,3,4,5,6,7,8,9,10,11,12,13]) self.__get_weeks_invalid_year_test()
def get(self): if self.request.get("flush")=="true": memcache.flush_all() #@UndefinedVariable resultDict = {"result": 0} self.response.out.write(json.dumps(resultDict)) else: self.response.out.write(errorMsg(202))
def test_invalid_weeks_with_populated_cache(self): logging.info("invalid week populated cache testing...") memcache.flush_all() self.__load_memcache_with_all_weeks() self.__test_invalid_week_query(year=2012,week_number=14) self.__test_invalid_week_query(year=2012,week_number=0) self.__test_invalid_week_query(year=1776,week_number=1)
def DELETE(self, request): if not web.ctx.env['REMOTE_ADDR'] in self.allowFlushFrom and not users.is_current_user_admin(): raise web.Forbidden() if request.split('/').pop() == '__ALL__': if 'memcache' in web.ctx.query: memcache.flush_all() return 'memcache flushed.\n' # entity selection is limited by 1000 but often timeout limit = 800 batch = [] for entity in Cache.all(keys_only=True).order('-expires').fetch(limit): batch.append(entity) n = len(batch) if n == 0: return 'No entries.\n' # batch deletion is limited by 500 but it timeouts above ~200 step = 200 if step > n: step = n for i in range(0, limit, step): db.delete(batch[i:i+step]) return '%s entries flushed\n' % n if self.ignoreQueryString is False: request += web.ctx.query cache = self.cache.get_by_key_name(request) if cache: cache.delete() memcache.delete(request) return 'ok\n'
def post(self, args, format): type = args.split("/")[0] key = args.split("/")[1].split("?")[0] if type.capitalize() in globals(): cls = globals()[type.capitalize()] if cls: values = {} values["key"] = key for k in self.request.arguments(): logging.info(k) value = self.request.get_all(k) logging.info(value) if k.split('-')[-1] in cls().properties().keys(): if ".ListProperty" in cls().properties()[k.split('-')[-1]].__class__.__str__(""): if k.split("-")[-1] == "permissions": values[k.split('-')[-1]] = self.request.get_all(k) else: values[k.split('-')[-1]] = [x.lstrip().rstrip() for x in value.split(",")] else: values[k.split('-')[-1]] = value values[k] = self.request.get_all(k) result = cls.update(values) if result: memcache.flush_all() if format == 'html': self.redirect(self.request.get("return_url")) elif format == 'json': self.json_out(result) else: self.response.out.write("Failed to update") else: self.response.out.write(self.request.get("return_url"))
def flush_cache(): """Flushes the memcache key used by this datamodel.""" memcache.flush_all() memcache.delete(IS_STOPPED_MEMCACHE_KEY) # Wait for a second before accessing the memcache again (in case there is a # propagation delay). time.sleep(1)
def post(self, type, format): if type.capitalize() in globals(): cls = globals()[type.capitalize()] if cls: values = {} for k in self.request.arguments(): value = self.request.get_all(k) if k.split('-')[-1] == "permissions": values[k.split('-')[-1]] = ",".join(self.request.get_all("page.permissions")) if k.split('-')[-1] in cls().properties() and "List" in cls().properties()[k.split('-')[-1]].__class__().__str__(): values[k.split('-')[-1]] = [x.lstrip().rstrip() for x in value] else: values[k.split('-')[-1]] = value values[k] = self.request.get(k) result = cls.create(values) if result: memcache.flush_all() if format == 'html': self.redirect(self.request.get("return_url")) elif format == 'json': self.json_out(result) else: self.response.out.write("Failed to update") else: self.response.out.write(self.request.get("return_url"))
def post(self): user = users.GetCurrentUser() if (not user): self.respondWithTemplate('please-sign-in.html', {'action': 'edit Radars'}) else: id = self.request.get("id") radar = Radar.get_by_id(int(id)) if not radar: self.respondWithText('Invalid Radar id') elif radar.user != user: self.respondWithText('Only the owner of a Radar can edit it') else: radar.title = self.request.get("title") radar.number = self.request.get("number") radar.number_intvalue = int(self.request.get("number")) radar.status = self.request.get("status") radar.description = self.request.get("description") radar.resolved = self.request.get("resolved") radar.product = self.request.get("product") radar.classification = self.request.get("classification") radar.reproducible = self.request.get("reproducible") radar.product_version = self.request.get("product_version") radar.originated = self.request.get("originated") radar.modified = datetime.datetime.now() radar.put() memcache.flush_all() self.redirect("/myradars")
def post(self): """Adds a new builder status message.""" # Check if this is a delete builder request. delete_builder = self.request.get('delete') if delete_builder: BuilderStatus.delete_builder(delete_builder) else: message = self.request.get('message') builder_name = self.request.get('builder_name') # If an entry with the builder_name already exists then delete it first. BuilderStatus.delete_builder(builder_name) # Add a new BuilderStatus entry. BuilderStatus(username=self.user.email(), builder_name=builder_name, message=message).put() # Flush the cache. memcache.flush_all() memcache.delete('builder_statuses') # Wait for a second before accessing the memcache again (in case there is a # propagation delay). time.sleep(1) # Set template values and load the template. self._handle()
def do_index(self): date = self.request.get('date') if date == backwards_date: date = get_prev_back_date() #don't index too much if date.year < 2013 and date.month < 9: return else: try: date = datetime.datetime.strptime(date, '%d.%m.%Y') except: logging.warning('bad date in request, assuming current') date = day_date(datetime.datetime.now()) logging.info('INDEXING WITH DATE %s' % date) end = date + datetime.timedelta(days=1) query = db.GqlQuery( 'SELECT * FROM RSSEntry WHERE date >= :1 AND date <= :2', date, end) entries = list(query) logging.info('start indexing %d entries' % len(entries)) for entry in entries: b = filters.scan(entry.desc) if b: logging.info('bad : %s' % b) keyname = rss.keyname_from_link(entry.url) bad = models.RSSBadEntry.get_or_insert(keyname, url=entry.url, desc=entry.desc, bad=b, date=entry.date) memcache.flush_all() keyname = str(date) models.IndexedDate.get_or_insert(keyname, date=date)
def post(self): action = self.request.get("admin_action") if (action == 'flush_cache'): memcache.flush_all() self.response.out.write(url_tools.dict_to_s(memcache.get_stats()))
def get(self): #delete cash memcache.flush_all() #redirect to blog FrontPage self.redirect('/')
def SnowFallGraphCreaterProcess(self): logging.info('Running the SnowFallGraphCreaterProcess.') ## TODO(wferrell): commented out to stop the error messages. #CreateSnowFallGraph() logging.info('SUCCESS: Running the SnowFallGraphCreaterProcess.') memcache.flush_all() logging.info('memcache.flush_all() run.')
def copy_get(self, url, config): if url == 'flushcache': memcache.flush_all() self.response.out.write('cache clear') return if hasattr(config, 'NOTURL') and url in config.NOTURL: gourl = config.SITE else: gourl = '%s/%s' % (config.SITE, url) if self.request.query_string: gourl = '%s?%s' % (gourl, self.request.query_string) origPostData = self.request.body page = None#memcache.get(gourl) if page is None or origPostData is not None: page = self.fetchurl(gourl, origPostData) ext = url.rsplit('\\.', 1)[-1].lower() if ext not in ['gif', 'jpg', 'png', 'bmp']: for k,v in config.REPLACE: page.content = page.content.replace(k,v) for func in config.funcs: if callable(func): page.content = func(page.content) if len(page.content) < 500000 and origPostData is None: memcache.set(gourl, page, 864000) self.set_status(page.status_code) self.setHeader(page, config) self.response.out.write(page.content)
def get(self): from geomancer.model import Cache, Locality, Clause, Georef from google.appengine.ext import ndb from google.appengine.api import memcache for x in [Cache, Locality, Clause, Georef]: ndb.delete_multi(x.query().iter(keys_only=True)) memcache.flush_all()
def post(self): user = users.GetCurrentUser() if (not user): self.respondWithDictionaryAsJSON({"error":"you must authenticate to add radars"}) else: title = self.request.get("title") number = self.request.get("number") status = self.request.get("status") description = self.request.get("description") product = self.request.get("product") classification = self.request.get("classification") reproducible = int(self.request.get("reproducible")) product_version = self.request.get("product_version") originated = self.request.get("originated") radar = Radar(title=title, number=number, user=user, status=status, description=description, resolved=resolved, product=product, classification=classification, reproducible=reproducible, product_version=product_version, originated=originated, created=datetime.datetime.now(), modified=datetime.datetime.now()) radar.put() memcache.flush_all() response = {"result": {"title":title, "number":number, "status":status, "description":description}} self.respondWithDictionaryAsJSON(response)
def recordLink(l, t, user_account): u = models.LinkModel(link = l, title = t, score = 0, submitter_id = user_account.getUserID(), submitter_username= user_account.getUsername()) u.put() logging.info("Recorded the link to the database, now flush and updating cache") memcache.flush_all() queryLinks() # calling query links automatically updates the cache. return "OK"
def post(self): user = users.get_current_user() if not user or 'user_id' not in dir(user): self.redirect(users.create_login_url('/addbuffr')) apiAddress = self.request.get('apiAddress') to_console = {} to_console["apiAddress"] = apiAddress to_console["is_valid_url(apiAddress)"] = (is_valid_url(apiAddress) != None) buffr_instance = Buffr() buffr_instance.apiName = self.request.get('apiName') buffr_instance.apiAddress = apiAddress APIUnstable = self.request.get('APIUnstable') if APIUnstable not in [True, False]: buffr_instance.APIUnstable = False else: buffr_instance.APIUnstable = APIUnstable buffr_instance.user_id = user.user_id() buffr_instance.user_email = user.email() buffr_instance.update_interval = int(self.request.get('updateInterval')) for possibility in user_readable_convertion_table: logging.info(str((possibility[0], buffr_instance.update_interval))) if int(possibility[0]) == buffr_instance.update_interval: buffr_instance.user_readable_update_interval = possibility[2] buffr_instance.end_point = hashlib.md5('%s:%s' % (user.user_id(), apiAddress)).hexdigest() buffr_instance.last_known_data = None buffr_instance.buffr_version = current_api_version buffr_instance.put() memcache.flush_all() logging.info('Added new Buffr to datastore') taskqueue.add(url='/confirm_working_url', params={'key': buffr_instance.key()}) render(self, 'addbuffer.html', {'to_console': to_console, 'submitted': True, 'apiAddress': apiAddress})
def CmdInitUserFromDbDo(from_name,msg): if not funcs.isAdmin(from_name): return 'you are not administrator' users_data = {} all_users = DBUser.all().fetch(1000) users_online_list = [] users_onlinemsgon_list = [] users_onlinemsgoff_list = [] users_offlinemsgon_list = [] users_offlinemsgoff_list = [] for user in all_users: users_data[user.gtalk] = user if xmpp_api.isOnline(user.gtalk): users_online_list.append(user.gtalk) if user.recvOnlineMsg: users_onlinemsgon_list.append(user.gtalk) else: users_onlinemsgoff_list.append(user.gtalk) if user.recvOfflineMsg: users_offlinemsgon_list.append(user.gtalk) else: users_offlinemsgoff_list.append(user.gtalk) memcache.flush_all() memcache.set('cache_all_user_list',users_data,43200) funcs.setConfigValueWithCache('cache_online_user_list',users_online_list) funcs.setConfigValueWithCache('cache_online_msgon_user_list',users_onlinemsgon_list) funcs.setConfigValueWithCache('cache_online_msgoff_user_list',users_onlinemsgoff_list) funcs.setConfigValueWithCache('cache_offline_msgon_user_list',users_offlinemsgon_list) funcs.setConfigValueWithCache('cache_offline_msgoff_user_list',users_offlinemsgoff_list) return 'init cache from db ok'
def post(self, key, type): self.check_user() entry = models.Entry.get(key) status = decode(self.request.get("status")) text = decode(self.request.get("text")) if type == "prod": if entry.prod_design_review != status: entry.prod_design_review = status new_story( self, "changed product design review status to <em>%s</em>" % status, entry=entry) entry.put() elif type == "eng": if entry.eng_design_review != status: entry.eng_design_review = status new_story( self, "changed eng design review status to <em>%s</em>" % status, entry=entry) entry.put() elif type == "mktg": if entry.mktg_review != status: entry.mktg_review = status new_story( self, "changed marketing review status to <em>%s</em>" % status, entry=entry) if text: new_story(self, text, entry=entry, type="comment") memcache.flush_all() self.redirect("/entries/" + key)
def sync_datastore(): dropbox = dropbox_api() dstore = {} for p in models.Picture.all(): dstore[p.path] = p.rev remote_pics = dropbox.request_meta('/pics') accepted = ['jpeg', 'jpg', 'png', 'gif', 'bmp'] for remote in remote_pics.get('contents', []): file_type = re.search(r'[^\.]*$', remote['path']).group(0).lower() if file_type in accepted: dstore_rev = dstore.get(remote['path'], "") if dstore_rev != remote['rev']: taskqueue.add(url='/sync/upload', params={ 'path': remote['path'], 'rev': remote['rev'], }) if remote['path'] in dstore: dstore.pop(remote['path']) for deleted in dstore: rec = models.Picture.all().filter('path =', deleted).get() rec.delete() memcache.flush_all() return True
def post(self): logging.info('Deleting ContentEntry') incremental_delete(model.ContentEntry.query().iter(keys_only=True), ndb.delete_multi_async) gs_bucket = config.settings().gs_bucket logging.info('Deleting GS bucket %s', gs_bucket) incremental_delete( (i[0] for i in gcs.list_files(gs_bucket)), lambda filenames: gcs.delete_files(gs_bucket, filenames)) logging.info('Flushing memcache') # High priority (.isolated files) are cached explicitly. Make sure ghosts # are zapped too. memcache.flush_all() logging.info('Finally done!')
def post(self, project_id): """ Handle modifying actions and redirect to a GET page. Args: project_id: A string specifyng the project ID. """ self.ensure_user_has_admin(project_id) if self.request.get('action:flush_memcache'): if memcache.flush_all(): message = 'Cache flushed, all keys dropped.' else: message = 'Flushing the cache failed. Please try again.' self.redirect(self._construct_url(remove=['action:flush_memcache'], add={'message': message})) elif self.request.get('action:delete_entities'): ds_access = DatastoreDistributed(project_id, DATASTORE_LOCATION, trusted=True) entity_keys = [datastore.Key(key) for key in self.request.params.getall('entity_key')] _delete_entities(ds_access, entity_keys) self.redirect(self._construct_url( remove=['action:delete_entities'], add={'message': '%d entities deleted' % len(entity_keys)})) else: self.error(404)
def flushcache(): if memcache.flush_all(): logging.info('Cache has been flushed') return True else: logging.info('Error flushing cache') return False
def setUp(self): # pylint: disable-msg=g-bad-name super(TestBase, self).setUp() memcache.flush_all() sites.ApplicationContext.clear_per_process_cache() self.auto_deploy = sites.ApplicationContext.AUTO_DEPLOY_DEFAULT_COURSE sites.ApplicationContext.AUTO_DEPLOY_DEFAULT_COURSE = ( self.get_auto_deploy()) self.supports_editing = False self.assert_default_namespace() self.namespace = '' self.base = '/' # Reload all properties now to flush the values modified in other tests. config.Registry.get_overrides(True)
def testTokenSaved(self): retry_params = api_utils.RetryParams(save_access_token=True) api = rest_api._RestApi('scope', retry_params=retry_params) t1 = api.get_token() self.assertNotEqual(None, t1) api = rest_api._RestApi('scope', retry_params=retry_params) t2 = api.get_token() self.assertEqual(t2, t1) memcache.flush_all() ndb.get_context().clear_cache() api = rest_api._RestApi('scope', retry_params=retry_params) t3 = api.get_token() self.assertEqual(t3, t1)
def savePost(post): for tagName in post.tags: tag = model.blog.Tag.get_by_key_name(ctrl.makeSlug(tagName)) if not tag: tag = model.blog.Tag(key_name=ctrl.makeSlug(tagName)) tag.name = tagName tag.postCount = 1 tag.put() else: pass if not post.slug: post.slug = ctrl.makeSlug(post.title) post.put() memcache.flush_all()
def perm_cached_class(func, flush=False): class_perm_cache = {} if flush: class_perm_cache = {} memcache.flush_all() def cached_func(self): cache_key = self.__name__ + "/" + func.__name__ if cache_key in class_perm_cache: return class_perm_cache[cache_key] data = memcache.get(cache_key) if data is None: data = func(self) memcache.add(cache_key, data) class_perm_cache[cache_key] = data return data return cached_func
def GET(self): result = {} action = web.input(action = None)['action'] if action=='quota': results = urlfetch.fetch('https://api.stackexchange.com/2.0/info?site=stackoverflow&key=%s' % api_key, headers = {'User-Agent': 'StackPrinter'}, deadline = 10) response = simplejson.loads(results.content) result['result'] = response if action=='quotaauth': results = urlfetch.fetch('https://api.stackexchange.com/2.0/info?site=stackoverflow&key=%s&access_token=%s' % (api_key, TokenManager.get_auth_token()), headers = {'User-Agent': 'StackPrinter'}, deadline = 10) response = simplejson.loads(results.content) result['result'] = response if action=='authkey': result['result'] = TokenManager.get_auth_token() elif action =='memcachestats': result = memcache.get_stats() elif action =='memcacheflush': result['result'] = memcache.flush_all() elif action =='normalize': deferred.defer(worker.deferred_normalize_printed_question) result['result'] = True elif action =='delete': service = web.input(service = None)['service'] question_id = web.input(question_id = None)['question_id'] result['printed_question_deletion'] = dbquestion.delete_printed_question(question_id,service) result['question_deletion'] = dbquestion.delete_question(question_id,service) result['answers_deletion'] = dbquestion.delete_answers(question_id,service) return render.admin(result)
def flush_mc(keys=None): if not (type(keys) is str or type(keys) is list or keys is None): raise TypeError max_retries = 50 if type(keys) is str: for _ in xrange(max_retries): if mc.delete(keys): break else: return False elif type(keys) is list: for _ in xrange(max_retries): if mc.delete_multi(keys): break else: return False else: for _ in xrange(max_retries): if mc.flush_all(): break else: return False return True
def update_from_tepco(): lastmod = Config.get_by_key_name('last-modified') lastmod = lastmod and lastmod.value data = tepco.from_web(lastmod) if not data: return '' # not yet updated Config(key_name='last-modified', value=data['lastmodstr']).put() if data.has_key('quick'): Config(key_name='quick.txt', value=data['quick']).put() # the image is updated hourly just after the hour. jst = jst_from_utc(data['usage-updated']) - datetime.timedelta(hours=1) jst = jst.replace(minute=0, second=0, microsecond=0) for hour, (usage, saving, forecast) in data['usage'].iteritems(): entryfor = utc_from_jst(jst.replace(hour=hour)) entry = Usage.all().filter('entryfor =', entryfor).get() if entry: entry.saving = saving entry.forecast = forecast if entry.usage != usage: entry.usage = usage entry.usage_updated = data['usage-updated'] if entry.capacity != data['capacity']: entry.capacity = data['capacity'] entry.capacity_updated = data['capacity-updated'] else: entry = Usage(entryfor=entryfor, year=entryfor.year, month=jst.month, day=jst.day, hour=hour, saving=saving, forecast=forecast, usage=usage, usage_updated=data['usage-updated'], capacity=data['capacity'], capacity_updated=data['capacity-updated']) entry.capacity_peak_period = data['capacity-peak-period'] entry.forecast_peak_usage = data['forecast-peak-usage'] entry.forecast_peak_period = data['forecast-peak-period'] entry.forecast_peak_updated = data['forecast-peak-updated'] entry.put() memcache.flush_all() return ''
def setUp(self): super(TestSetupMixin, self).setUp() from google.appengine.api import apiproxy_stub_map from google.appengine.api import memcache from google.appengine.api import queueinfo from google.appengine.datastore import datastore_stub_util from google.appengine.ext import testbed from google.appengine.ext.testbed import TASKQUEUE_SERVICE_NAME before_level = logging.getLogger().getEffectiveLevel() os.environ['APPLICATION_ID'] = self.TEST_APP_ID os.environ['CURRENT_VERSION_ID'] = self.TEST_VERSION_ID os.environ['HTTP_HOST'] = '%s.appspot.com' % self.TEST_APP_ID os.environ['DEFAULT_VERSION_HOSTNAME'] = os.environ['HTTP_HOST'] os.environ['CURRENT_MODULE_ID'] = 'foo-module' try: logging.getLogger().setLevel(100) self.testbed = testbed.Testbed() self.testbed.activate() self.testbed.setup_env(app_id=self.TEST_APP_ID, overwrite=True) self.testbed.init_memcache_stub() hr_policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy( probability=1) self.testbed.init_datastore_v3_stub(consistency_policy=hr_policy) self.testbed.init_taskqueue_stub() root_path = os.path.realpath(os.path.dirname(__file__)) # Actually need to flush, even though we've reallocated. Maybe because the # memcache stub's cache is at the module level, not the API stub? memcache.flush_all() finally: logging.getLogger().setLevel(before_level) define_queues = ['other'] taskqueue_stub = apiproxy_stub_map.apiproxy.GetStub('taskqueue') taskqueue_stub.queue_yaml_parser = ( lambda x: queueinfo.LoadSingleQueue( 'queue:\n- name: default\n rate: 1/s\n' + '\n'.join( '- name: %s\n rate: 1/s' % name for name in define_queues)))
def cache_it(dbkey = 'joke'): contents = memcache.get(dbkey) if not contents: global query_time query_time = datetime.now() if dbkey is not 'joke': memcache.flush_all() contents = Content.get_by_id(int(dbkey)) memcache.set(dbkey, contents) else: contents = db.GqlQuery("select * from Content order by created desc") contents = list(contents) memcache.set(dbkey, contents) this_moment = datetime.now() global time_elapsed time_elapsed = int((this_moment - query_time).total_seconds()) return contents, time_elapsed
def testMemcacheClear(self): LoadInfo.InitializeTable() LoadInfo.AddInstance('test-instance') LoadInfo.RegisterInstanceIpAddress('test-instance', '1.2.3.4') LoadInfo.UpdateLoadInfo('test-instance', 55) # Simulate loss of all data in Memcache. memcache.flush_all() LoadInfo.UpdateLoadInfo('test-instance', 38) self.assertEqual( { 'test-instance': { 'ip_address': '1.2.3.4', 'load': 38, 'force': False } }, LoadInfo.GetAll())
def signin(cls, match_id, code): match = Match.getone(match_id) if datetime.now() < match.signinEarliest: logging.debug("You are too early for sign in") return {"status": False, "reason": "You are too early for sign in", "code": -1} if datetime.now() > match.signinLatest: logging.debug("You are too late for sign in") logging.debug("Sign in user {}".format(current_user.key_id)) match.signin(current_user.key_id) play = Play.getbyMatchPeople(match_id, current_user.key_id) if play is None: logging.debug("this guy didn't sign up, but is sign-in now") match.signup(current_user.key_id) Play.create(current_user.key_id, match_id, missing=True) else: play.signinTime = datetime.now() play.put() memcache.flush_all() return {"status": False, "reason": "You are too late for sign in", "code": 1} people = People.getone(current_user.key_id) if people and people.key in match.participatedPeople: logging.debug("You have already signed in") return {"status": False, "reason": "You have already signed in", "code": 0} if match and code == match.signinCode: logging.debug("Sign in user {}".format(current_user.key_id)) match.signin(current_user.key_id) play = Play.getbyMatchPeople(match_id, current_user.key_id) if play is None: logging.debug("this guy didn't sign up, but is sign-in now") match.signup(current_user.key_id) Play.create(current_user.key_id, match_id, missing=True) else: play.signinTime = datetime.now() play.put() memcache.flush_all() return {"status": True} return {"status": False}
def post(self): clear = memcache.flush_all() if clear: logging.info("Cache cleared") else: logging.error("Problem clearing cache") self.redirect("/admin/")
def post(self): subject = self.request.get("subject") content = self.request.get("content") if subject and content: p = Post(parent=blog_key(), subject=subject, content=content) p.put() get_posts(True) memcache.flush_all() self.redirect('/blog/%s' % str(p.key().id())) else: error = "subject and content, please!" self.render("newpost.html", subject=subject, content=content, error=error, s=self)
def page_delete_user(): user_id = request.args.get("user") if user_id: user = ndb.key.Key(urlsafe=user_id).get() user.delete() app.logger.info("User deleted") memcache.flush_all() msg = [] for user in User.query().fetch(): msg.append(u"""<a href="{url}?feed={user}">{uid} {email}</a>""".format( url=url_for('.page_delete_user'), user=user.key.urlsafe(), uid=user.key.id(), email=user.email)) return u"<br>".join(msg)
def get(self): template = JINJA_ENVIRONMENT.get_template('index.html') current_user = users.get_current_user() db_user = User.query(User.email == current_user.email()).fetch() if not db_user: new_user = User() new_user.email = current_user.email() new_user.put() memcache.flush_all() values = { "users": User().query().fetch(), "items": Item().query().fetch() } self.response.write(template.render(values))
def post(self): """Handle modifying actions and/or redirect to GET page.""" super(MemcacheViewerRequestHandler, self).post() next_param = {} if self.request.get('action:flush'): if memcache.flush_all(): next_param['message'] = 'Cache flushed, all keys dropped.' else: next_param['message'] = 'Flushing the cache failed. Please try again.' elif self.request.get('action:display'): next_param['key'] = self.request.get('key') elif self.request.get('action:edit'): next_param['edit'] = self.request.get('key') elif self.request.get('action:delete'): key = self.request.get('key') result = memcache.delete(key) if result == memcache.DELETE_NETWORK_FAILURE: next_param['message'] = ('ERROR: Network failure, key "%s" not deleted.' % key) elif result == memcache.DELETE_ITEM_MISSING: next_param['message'] = 'Key "%s" not in cache.' % key elif result == memcache.DELETE_SUCCESSFUL: next_param['message'] = 'Key "%s" deleted.' % key else: next_param['message'] = ('Unknown return value. Key "%s" might still ' 'exist.' % key) elif self.request.get('action:save'): key = self.request.get('key') value = self.request.get('value') type_ = self.request.get('type') next_param['key'] = key converter = self.FRIENDLY_TYPE_NAME_TO_CONVERTER[type_] try: memcache_value = converter.to_cache(value) except ValueError as e: next_param['message'] = 'ERROR: Failed to save key "%s": %s.' % (key, e) else: if self._set_memcache_value(key, memcache_value, converter.memcache_type): next_param['message'] = 'Key "%s" saved.' % key else: next_param['message'] = 'ERROR: Failed to save key "%s".' % key elif self.request.get('action:cancel'): next_param['key'] = self.request.get('key') else: next_param['message'] = 'Unknown action.' next = self.request.path_url if next_param: next = '%s?%s' % (next, self._urlencode(next_param)) self.redirect(next)
def clear(*args, **kwargs): """Removes all entities from the datastore. """ # TODO(dbentley): If there are more than 1000 instances of any model, # this method will not clear all instances. Instead, it should continually # call .all(), delete all those, and loop until .all() is empty. entities = itertools.chain(*[ Survey.all(), SurveyRecord.all(), GCIOrganization.all(), GSoCTimeline.all(), GCITimeline.all(), GSoCProgram.all(), GSoCProject.all(), GSoCProposal.all(), GCIProgram.all(), GCIScore.all(), GSoCStudentInfo.all(), GCIStudentInfo.all(), GCITask.all(), Sponsor.all(), Site.all(), Document.all(), # The below models are all subclasses of ndb.Model and therefore must # use .query() to return all instances instead of .all(). soc_org_model.SOCOrganization.query(), profile_model.Profile.query(), soc_profile.SOCStudentData.query(), user.User.query(), address.Address.query(), contact.Contact.query() ]) try: for entity in entities: if isinstance(entity, ndb.Model): entity.key.delete() else: entity.delete() except db.Timeout: return http.HttpResponseRedirect('#') memcache.flush_all() return http.HttpResponse('Done')
def post(self): self._require_admin() flushed = list() if self.request.get("all_keys") == "all_keys": memcache.flush_all() flushed.append("all memcache values") if self.request.get("webcast_keys") == "webcast_keys": flushed.append(MemcacheWebcastFlusher.flush()) if self.request.get('memcache_key') is not "": memcache.delete(self.request.get("memcache_key")) flushed.append(self.request.get("memcache_key")) if self.request.get('event_key') is not "": # Clear event page and event teams pages event_key = self.request.get('event_key') memcache_keys_to_delete = [ EventDetail().cache_key.format(event_key) ] event = Event.get_by_id(event_key) for team in event.teams: memcache_keys_to_delete.append(TeamDetail().cache_key.format( team.key.id(), event.year)) memcache_keys_to_delete.append( TeamCanonical().cache_key.format(team.key.id())) memcache.delete_multi(memcache_keys_to_delete) flushed += memcache_keys_to_delete if self.request.get('return_url') is not "": self.redirect("{}?flushed={}".format( self.request.get('return_url'), flushed)) return self.template_values.update({ "flushed": flushed, "memcache_stats": memcache.get_stats(), }) path = os.path.join(os.path.dirname(__file__), '../../templates/admin/memcache_index.html') self.response.out.write(template.render(path, self.template_values))
def store_bot_config(content): """Stores a new version of bot_config.py.""" out = VersionedFile(content=content).store('bot_config.py') # Clear the cached version value since it has now changed. This is *super* # aggressive to flush all memcache but that's the only safe way to not send # old code by accident. while not memcache.flush_all(): pass return out
def _save(self, cd, project): new_name = cd['name'].strip() # TODO(joeo): Big race condition here. # in_use = models.Project.get_project_for_name(new_name) if in_use and project.key() != in_use.key(): self.errors['name'] = ['Name is already in use'] if self.is_valid(): project.name = new_name project.comment = cd['comment'] project.owners_users, project.owners_groups = \ fields.UserGroupField.get_user_and_group_keys(cd['owners']) project.set_code_reviews( _keys_for(map(_field_to_approval_right, cd['code_reviews']))) project.put() memcache.flush_all()
def test_atom_format(self): for test_module in test_facebook, test_instagram, test_twitter: self.reset() memcache.flush_all() self.mox.StubOutWithMock(FakeSource, 'get_actor') FakeSource.get_actor(None).AndReturn(test_module.ACTOR) self.activities = [copy.deepcopy(test_module.ACTIVITY)] # include access_token param to check that it gets stripped resp = self.get_response('/fake?format=atom&access_token=foo&a=b') self.assertEquals(200, resp.status_int) self.assert_multiline_equals( test_module.ATOM % { 'request_url': 'http://localhost/fake?format=atom&access_token=foo&a=b', 'host_url': 'http://fa.ke/', 'base_url': 'http://fa.ke/', }, resp.body)
def get(self, table): if table in dir(kloombaDb): i = 0 request = GqlQuery('SELECT * FROM ' + table).run() for item in request: i += 1 if 'timestamp' in dir(item): kloombaDb.__dict__[table].__dict__['timestamp'].__dict__[ 'auto_now'] = False if 'lastActiveTimestamp' in dir(item): kloombaDb.__dict__[table].__dict__[ 'lastActiveTimestamp'].__dict__['auto_now'] = False item.put() memcache.flush_all() self.response.headers['Content-Type'] = 'text/plain' self.response.out.write("Updated table: %s\nUpdated entitys: %d" % (table, i))
def get(self): # enforce rate limit if ratelimit(os.environ['REMOTE_ADDR']) is False: # logging.warning("ERROR: Daily request quota exceeded for %s" % os.environ['REMOTE_ADDR']) self.response.set_status(403) self.response.out.write( "%s daily requests quota exceeded. If you're interested in a higher quota, contact [email protected]\n" % CYCLE_REQ_QUOTA) return cache_key = "response-%s" % self.request.url # GAE takes care of version namespacing internally url = 'http://' + self.request.get('url').replace( 'http://', '') # handle 'tinyurl.com' as well as 'http://tinyurl.com' callback = self.request.get('callback') format = self.request.get('format') flush = self.request.get('flushcache') if flush == 'now': memcache.flush_all() if url == 'http://': self.homepage() else: try: out = memcache.get(cache_key) if out is None: if format == 'json': out = self.json(url, callback) else: out = self.plain(url) memcache.add(cache_key, out) self.response.headers.add_header('Access-Control-Allow-Origin', '*') self.response.out.write(out) except Exception, err_msg: logging.warning("Exception in get(): %s" % err_msg) self.response.set_status(404) self.response.out.write('error')
def updateItem(client_id, cart_id, item_id): try: client_key = ndb.Key(Clients, client_id) cart_key = ndb.Key(Carts, cart_id) item_key = ndb.Key(Items, item_id) except: abort(404) item = item_key.get() item.name = request.json.get('name', item.name) item.put() client = client_key.get() cart = cart_key.get() memcache.flush_all() return make_response( jsonify({ 'Client_Email': client.email, 'Cart_name': cart.name, 'updated': item.to_dict() }), 200)
def get(self): """When the user grants access, they are redirected back to this handler where their authorized request token is exchanged for a long-lived access token.""" client = gdata.apps.client.AppsClient(domain="wandoujia.com") client.ssl = True current_user = users.get_current_user() # Remember the token that we stashed? Let's get it back from # datastore now and adds information to allow it to become an # access token. request_token_key = 'request_token_%s' % current_user.email() request_token = gdata.gauth.ae_load(request_token_key) gdata.gauth.authorize_request_token(request_token, self.request.uri) # We can now upgrade our authorized token to a long-lived # access token by associating it with gdocs client, and # calling the get_access_token method. client.auth_token = client.get_access_token(request_token) # Note that we want to keep the access token around, as it # will be valid for all API calls in the future until a user # revokes our access. For example, it could be populated later # from reading from the datastore or some other persistence # mechanism. access_token_key = 'access_token_%s' % current_user.email() gdata.gauth.ae_save(request_token, access_token_key) # Write user info into database user_name = current_user.email().split('@')[0] logging.info(user_name) person = models.Person().get_by_key_name(user_name) if not person: person = models.Person(gaia=current_user, email=current_user.email(), key_name=user_name) person.is_setup = True person.put() memcache.flush_all() self.redirect('/')
def setup_for_testing(require_indexes=True, define_queues=[]): """Sets up the stubs for testing. Args: require_indexes: True if indexes should be required for all indexes. define_queues: Additional queues that should be available. """ from google.appengine.api import apiproxy_stub_map from google.appengine.api import memcache from google.appengine.api import queueinfo from google.appengine.datastore import datastore_stub_util from google.appengine.tools import old_dev_appserver from google.appengine.tools import dev_appserver_index before_level = logging.getLogger().getEffectiveLevel() try: logging.getLogger().setLevel(100) root_path = os.path.realpath(os.path.dirname(__file__)) old_dev_appserver.SetupStubs( TEST_APP_ID, root_path=root_path, login_url='', datastore_path=tempfile.mktemp(suffix='datastore_stub'), history_path=tempfile.mktemp(suffix='datastore_history'), blobstore_path=tempfile.mktemp(suffix='blobstore_stub'), require_indexes=require_indexes, clear_datastore=False) dev_appserver_index.SetupIndexes(TEST_APP_ID, root_path) # Actually need to flush, even though we've reallocated. Maybe because the # memcache stub's cache is at the module level, not the API stub? memcache.flush_all() finally: logging.getLogger().setLevel(before_level) datastore_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3') hr_policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=1) datastore_stub.SetConsistencyPolicy(hr_policy) taskqueue_stub = apiproxy_stub_map.apiproxy.GetStub('taskqueue') taskqueue_stub.queue_yaml_parser = ( lambda x: queueinfo.LoadSingleQueue( 'queue:\n- name: default\n rate: 1/s\n' + '\n'.join('- name: %s\n rate: 1/s' % name for name in define_queues)))
def setUpClass(cls): # Configuring AppEngine Testbed cls.testbed = testbed.Testbed() cls.testbed.activate() cls.policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy( probability=1) cls.testbed.init_datastore_v3_stub(consistency_policy=cls.policy) cls.testbed.init_memcache_stub() # Setting Ndb Cache ndb.get_context().set_cache_policy(False) ndb.get_context().clear_cache() # Flushing Memcache memcache.flush_all() # Creating Creating test objects cls.counter_name = 'test-counter' cls.counter_name_list = ['test-1', 'test-2', 'test-3']