def admin_CreateBackupOfDomainAndPages(req):
aa = getAllFromFromQuery(datamodel.DB_Domains.all())
bb = getAllFromFromQuery(datamodel.DB_Pages.all())
toIgnore = {'dateUpdated'}
res_domains = [DB_SimpleToDict(x, toIgnore) for x in aa]
res_pages = [DB_SimpleToDict(x, toIgnore) for x in bb]
return RetType.JSONSUCCESS, {'domains':res_domains, 'pages':res_pages}
def admin_CreateBackupZipForTag(req):
import cStringIO
import zipfile
def cleanDate(dd): return str(dd).replace(':', '.').replace(' ', '-')
whichTag = req.get('tag')
if not whichTag: return RetType.RAW, 'Must specify tag'
allFiles = getAllFromFromQuery(datamodel.DB_FileVersion.all().filter('tags =', whichTag))
allFiles.sort(key=lambda x:x.version, reverse=True)
zipstream = cStringIO.StringIO()
zf = zipfile.ZipFile(zipstream, 'w', zipfile.ZIP_DEFLATED)
alreadyWritten = set()
for x in allFiles:
if x.filename in alreadyWritten: continue
alreadyWritten.add(x.filename)
content = x.parent().data # reads data from datastore... slow
zf.writestr(str(x.filename), content)
zf.close()
res = zipstream.getvalue()
outfile = str('onlinefiles-%s-%s.zip' % (whichTag, cleanDate(datetime.datetime.now())))
return RetType.HEADERSANDRAW, {'Content-Type':'application/zip', 'Content-Disposition': 'attachment; filename="'+outfile+'"' }, res
def admin_getAllFileVersions(req):
a = getAllFromFromQuery(datamodel.DB_FileVersion.all().order('version'))
d = defaultdict(list)
for x in a: d[x.filename].append([x.version]+x.tags)
#for x in a: d[x.filename].append(x.version)
return RetType.JSONSUCCESS, {'files':d}
def admin_GetDomainList(req):
whichKey = req.get('key')
if whichKey:
all = datamodel.DB_Domains.get_by_id(int(whichKey)) # load a specific one
if not all: return RetType.JSONFAIL, {'text':'Domain not found by key'}
all = [all]
else:
all = getAllFromFromQuery(datamodel.DB_Domains.all())
res = [{'key':x.key().id(),'name':x.name, 'regex':x.regex, 'defaultTitle':x.defaultTitle, 'order':x.order, 'dateUpdated':str(x.dateUpdated)} for x in all]
return RetType.JSONSUCCESS, {'domains':res}
def admin_clearAllBuildData(req):
all = getAllFromFromQuery(datamodel.DB_FileBuild.all(keys_only=True))
for x in chunks(all, 1000):
db.delete(x)
nextVersionNum = datamodel.DB_JV_AtomicCounter.GetNextCounter(_fileVerKey)
from inmemconfig import InAppMemConfig
InAppMemConfig.ForceVersionIncrement()
return RetType.JSONSUCCESS
def admin_RestoreFromBackupZip(req, user):
import zipfile
file = os.path.join(os.path.dirname(__file__), 'zips/orig.zip')
zf = zipfile.ZipFile(file, 'r')
allFiles = getAllFromFromQuery(datamodel.DB_FileVersion.all().order('version'))
byName = {} # most recent version.
for x in allFiles: byName[x.filename] = x
extractStats = {}
filesInZip = zf.namelist()
forceFiles = req.get('forceFiles')
if forceFiles == '__all__':
forceFiles = set(filesInZip)
elif forceFiles:
forceFiles = set(forceFiles.split(','))
else:
forceFiles = set()
for fiz in filesInZip:
extractStats[fiz] = ''
data = zf.read(fiz)
new_hash = getGitBlobHash(data)
if fiz in byName:
curVer = byName[fiz]
oldHash = curVer.key().parent().name()
if oldHash == new_hash:
extractStats[fiz] += 'Same version already the head version in the system. '
continue
else:
extractStats[fiz] += 'Different version in zip file and system head. '
if fiz not in forceFiles: continue
theDataKey = datamodel.DB_FileContent.all(keys_only=True).filter('__key__ =', db.Key.from_path(u'DB_FileContent', new_hash)).get()
if not theDataKey:
theDataKey = datamodel.DB_FileContent(key_name = new_hash, data = db.Blob(data)).put()
extractStats[fiz] += 'Added to cache. '
else:
extractStats[fiz] += 'Already in cache. '
nextVersionNum = datamodel.DB_JV_AtomicCounter.GetNextCounter(_fileVerKey)
tags = ['z', 'a'] # make it head
newFileVer = datamodel.DB_FileVersion(parent=theDataKey, key_name=str(nextVersionNum),
filename=fiz,version=nextVersionNum,uid=user.key().id(),tags=tags)
newFileVer.put()
extractStats[fiz] += 'Added version %s with hash: %s ' % (nextVersionNum, new_hash)
#forceCacheRebuild(fiz, 'z')
zf.close()
return RetType.JSONSUCCESS, {'stats':extractStats}
def admin_CreateAllFileZip(req):
import cStringIO
import zipfile
zipstream = cStringIO.StringIO()
zf = zipfile.ZipFile(zipstream, 'w', zipfile.ZIP_DEFLATED)
allFiles = getAllFromFromQuery(datamodel.DB_FileVersion.all())
def cleanDate(dd): return str(dd).replace(':', '.').replace(' ', '-')
for x in allFiles:
content = x.parent().data
aa = x.filename.rsplit('.', 1)
aa.insert(1, '%s_%s%s' % (str(x.version).zfill(4), cleanDate(x.dateAdded), '_a' if 'a' in x.tags else '') )
tfn = '.'.join(aa)
zf.writestr(str(tfn), content)
zf.close()
outfile = 'onlinefiles-' + cleanDate(datetime.datetime.now()) + '.zip'
res = zipstream.getvalue()
return RetType.HEADERSANDRAW, {'Content-Type':'application/zip', 'Content-Disposition': 'attachment; filename="'+outfile+'"' }, res