def get_changed_entities(self):
# get classes
# not blacklisted
klass_names = [
k.kind_name
for k in Kind.all()
if k.kind_name not in self.blacklist
]
# check for cases where the Klass cannot be converted (KeyError)
# this happens in production for reasons I don't understand
# bmh 2013
Klasses = []
for k in klass_names:
try:
Klass = kind_to_class(k)
except AttributeError:
pass
else:
Klasses.append(Klass)
# get entites
if not self.last_check:
self.last_check = 0
entities = [
e
for Klass in Klasses
for e in Klass.all().filter(
"modified > ", self.last_check
).order("modified").fetch(self.max_entities_to_index)
]
return entities
def get(self):
# make list of all Kinds
self.response.headers["Content-Type"] = "text/plain"
self.response.out.write("Kinds:\n")
kind_names = []
q = Kind.all()
for kind in q.fetch(100):
if kind.kind_name[0] != "_":
kind_names.append(kind.kind_name)
self.response.out.write(kind.kind_name + "\n")
# excute backup queue in backup-builtin-queue
params = {
"name": "SchduledBackup",
"queue": "backup-builtin-queue",
"filesystem": "blobstore",
"kind": kind_names
}
url = "/_ah/datastore_admin/backup.create"
taskqueue.add(url=url,
params=params,
queue_name="backup-builtin-queue")
self.response.out.write("end")
def get_all_datastore_kinds():
# Start with unrestricted kind query
q = Kind.all()
# Print query results
for k in q:
if not k.kind_name.startswith('_'):
yield k.kind_name
def get_all_datastore_kinds():
# Start with unrestricted kind query
q = Kind.all()
# Print query results
for k in q:
if not k.kind_name.startswith('_'):
yield k.kind_name
def deleteAllEntities(): from models import models q = Kind.all() model_types = [k.kind_name for k in q] for model_type in model_types: model_type = getattr(models, model_type) model_type_objects = model_type.query().fetch() for obj in model_type_objects: obj.key.delete()
def testKindsRenderingFromStats(self):
"""Test list of kinds renders in list_actions with kinds timing out."""
self.mox = mox.Mox()
self.mox.StubOutWithMock(Kind, 'all')
Kind.all().AndRaise(db.Timeout('Timeout'))
self.mox.ReplayAll()
try:
stat_test_kinds = ['test1', 'test2', 'test3']
self.CreateStatEntity(stats.GlobalStat.STORED_KIND_NAME)
for kind in stat_test_kinds:
self.CreateStatEntity(stats.KindStat.STORED_KIND_NAME, kind)
self.handler.get()
kind_list = [stat['kind_name'] for stat in self.params['kind_stats']]
self.assertListEqual(stat_test_kinds, kind_list)
self.mox.VerifyAll()
finally:
self.mox.UnsetStubs()
def deleteAllEntities():
from models import models
q = Kind.all()
model_types = [k.kind_name for k in q]
for model_type in model_types:
model_type = getattr(models, model_type)
model_type_objects = model_type.query().fetch()
for obj in model_type_objects:
obj.key.delete()
def testKindsRenderingFromStats(self):
"""Test list of kinds renders in list_actions with kinds timing out."""
self.mox = mox.Mox()
self.mox.StubOutWithMock(Kind, 'all')
Kind.all().AndRaise(db.Timeout('Timeout'))
self.mox.ReplayAll()
try:
stat_test_kinds = ['test1', 'test2', 'test3']
self.CreateStatEntity(stats.GlobalStat.STORED_KIND_NAME)
for kind in stat_test_kinds:
self.CreateStatEntity(stats.KindStat.STORED_KIND_NAME, kind)
self.handler.get()
kind_list = [
stat['kind_name'] for stat in self.params['kind_stats']
]
self.assertListEqual(stat_test_kinds, kind_list)
self.mox.VerifyAll()
finally:
self.mox.UnsetStubs()
def backup(self, *args, **kwargs):
"""
Backup job kick-off for Google App Engine datastore admin.
Enable it on production systems when necessary enabling and pre-configuration is done.
You need to enable GAE datastore admin, initialize a GCS bucket for your app and install
another queue specified in queue.yaml to get this run. This is done using
``` gcloud app deploy -q --project=YOUR-PROJECT queue.yaml
Then, uncomment above decorator.
"""
from google.appengine.ext.db.metadata import Kind
from google.appengine.api.app_identity import get_application_id
from google.appengine.api import taskqueue
appname = get_application_id()
kinds = []
q = Kind.all()
for kind in q.fetch(100):
kindName = kind.kind_name
if kindName in ["SharedConfData"] or kindName.startswith("_"):
continue
kinds.append(kindName)
logging.debug("Adding kind '%s' to backup" % kindName)
taskqueue.add(
url="/_ah/datastore_admin/backup.create",
method="GET",
queue_name="data-backup",
params={
"name":
"daily_backup", #No ending "_" accepted here, this causes error 400????
"filesystem": "gs",
"gs_bucket_name": "%s.appspot.com/backup" % appname,
"kind": kinds
})
logging.info("Daily backup queued for %s" % ", ".join(kinds))
def clean(self):
deadline = datetime.datetime.now() - datetime.timedelta(seconds=60)
logging.info("number of namespaces: %s", Namespace.all().count())
for namespace in Namespace.all():
namespace_name = namespace.namespace_name
logging.info('found namespace "%s"', namespace_name)
if namespace_name[0:1] == "_":
logging.info("skip system namespace: %s", namespace_name)
continue
mo = re.match("^ns_([0-9]+)$", namespace_name)
if mo:
ns = UsedNamespace.get_by_id(int(mo.group(1)))
logging.info("UsedNamespace for namespace: %s", ns)
if ns:
if ns.created > deadline:
logging.info("Cannot delete namespace %s yet", namespace_name)
continue # this namespace is still in use
db.delete(ns)
# delete all kind in this namespace
logging.info('deleting all kinds from namespace "%s"', namespace_name)
namespace_manager.set_namespace(namespace_name)
for kind in Kind.all():
if namespace_name == "" and kind.kind_name == "UsedNamespace":
logging.info("not delete UsedNamespace kind")
elif kind.kind_name[0:1] == "_":
logging.info("skip system kind %s", kind.kind_name)
else:
logging.info("delete kind %s", kind.kind_name)
try:
self.delete_kind(kind.kind_name)
except Exception, e:
logging.info("Got error: %s", e)
logging.info("Try to create the model class %s", kind.kind_name)
code = "from google.appengine.ext import db\nclass %s(db.Model): pass" % kind.kind_name
compiled = compile(code, "create model", "exec")
namespace = {}
exec compiled in namespace
self.delete_kind(kind.kind_name)
def get(self):
# make list of all Kinds
self.response.headers["Content-Type"] = "text/plain"
self.response.out.write("Kinds:\n")
kind_names = []
q = Kind.all()
for kind in q.fetch(100):
if kind.kind_name[0] != "_":
kind_names.append(kind.kind_name)
self.response.out.write(kind.kind_name+"\n")
# excute backup queue in backup-builtin-queue
params = {
"name" : "SchduledBackup",
"queue" : "backup-builtin-queue",
"filesystem" : "blobstore",
"kind" : kind_names
}
url = "/_ah/datastore_admin/backup.create"
taskqueue.add(url=url, params=params,queue_name="backup-builtin-queue")
self.response.out.write("end")
def _get_all_datastore_kinds():
for kind in Kind.all():
if not kind.kind_name.startswith('_'):
yield kind.kind_name
def testGetKinds(self):
logger = getLogger().getChild(__name__)
Initialize("obomb")
q = Kind.all()
for kind in q.fetch(100):
logger.info(kind.kind_name)
from google.appengine.api import datastore
USERNAME = raw_input('Enter your Google app engine Username :'******'Enter your Google app engine password :'******'Enter your Applicaton id :')
APPLICATION_ID_SITE = APPLICATION_ID + '.appspot.com'
def auth_func():
return (USERNAME, PASSWORD)
remote_api_stub.ConfigureRemoteApi(None, '/_ah/remote_api', auth_func, APPLICATION_ID_SITE)
kind_result = []
kinds = Kind.all()
for values in kinds:
kind_result.append(str(values.kind_name))
#truncating to remove the builtin kinds
truncated_kind_result = [i for i in kind_result if i[:1] != '_']
# print truncated_kind_result
print "###############################################"
print "Your application authenticated"
def get(self):
arr = []
for k in Kind.all():
if k.kind_name.find('_Ah') != 0:
arr.append(k.kind_name)
self.success(arr)
def get_all_datastore_kinds():
for kind in Kind.all():
if not kind.kind_name.startswith('_'):
yield kind.kind_name