Example #1
0
def admin_CreateBackupOfDomainAndPages(req):
    aa = getAllFromFromQuery(datamodel.DB_Domains.all())
    bb = getAllFromFromQuery(datamodel.DB_Pages.all())
    toIgnore = {'dateUpdated'}
    res_domains = [DB_SimpleToDict(x, toIgnore) for x in aa]
    res_pages = [DB_SimpleToDict(x, toIgnore) for x in bb]
    return RetType.JSONSUCCESS, {'domains':res_domains, 'pages':res_pages}
Example #2
0
def admin_CreateBackupZipForTag(req):
    import cStringIO
    import zipfile
    def cleanDate(dd): return str(dd).replace(':', '.').replace(' ', '-')

    whichTag = req.get('tag')
    if not whichTag: return RetType.RAW, 'Must specify tag'

    allFiles = getAllFromFromQuery(datamodel.DB_FileVersion.all().filter('tags =', whichTag))
    allFiles.sort(key=lambda x:x.version, reverse=True)

    zipstream = cStringIO.StringIO()
    zf = zipfile.ZipFile(zipstream, 'w', zipfile.ZIP_DEFLATED)

    alreadyWritten = set()
    for x in allFiles:
        if x.filename in alreadyWritten: continue
        alreadyWritten.add(x.filename)
        content = x.parent().data # reads data from datastore... slow
        zf.writestr(str(x.filename), content)

    zf.close()
    res = zipstream.getvalue()
    outfile = str('onlinefiles-%s-%s.zip' % (whichTag, cleanDate(datetime.datetime.now())))
    return RetType.HEADERSANDRAW, {'Content-Type':'application/zip', 'Content-Disposition': 'attachment; filename="'+outfile+'"' }, res
Example #3
0
def admin_getAllFileVersions(req):
    a = getAllFromFromQuery(datamodel.DB_FileVersion.all().order('version'))

    d = defaultdict(list)
    for x in a: d[x.filename].append([x.version]+x.tags)
    #for x in a: d[x.filename].append(x.version)
    return RetType.JSONSUCCESS, {'files':d}
Example #4
0
def admin_GetDomainList(req):
    whichKey = req.get('key')
    if whichKey:
        all = datamodel.DB_Domains.get_by_id(int(whichKey)) # load a specific one
        if not all: return RetType.JSONFAIL, {'text':'Domain not found by key'}
        all = [all]
    else:
        all = getAllFromFromQuery(datamodel.DB_Domains.all())
    res = [{'key':x.key().id(),'name':x.name, 'regex':x.regex, 'defaultTitle':x.defaultTitle, 'order':x.order, 'dateUpdated':str(x.dateUpdated)} for x in all]
    return RetType.JSONSUCCESS, {'domains':res}
Example #5
0
def admin_clearAllBuildData(req):
    all = getAllFromFromQuery(datamodel.DB_FileBuild.all(keys_only=True))

    for x in chunks(all, 1000):
        db.delete(x)

    nextVersionNum = datamodel.DB_JV_AtomicCounter.GetNextCounter(_fileVerKey)

    from inmemconfig import InAppMemConfig
    InAppMemConfig.ForceVersionIncrement()

    return RetType.JSONSUCCESS
Example #6
0
def admin_RestoreFromBackupZip(req, user):
    import zipfile
    file = os.path.join(os.path.dirname(__file__), 'zips/orig.zip')
    zf = zipfile.ZipFile(file, 'r')

    allFiles = getAllFromFromQuery(datamodel.DB_FileVersion.all().order('version'))
    byName = {} # most recent version.
    for x in allFiles: byName[x.filename] = x

    extractStats = {}
    filesInZip = zf.namelist()

    forceFiles = req.get('forceFiles')
    if forceFiles == '__all__':
        forceFiles = set(filesInZip)
    elif forceFiles:
        forceFiles = set(forceFiles.split(','))
    else:
        forceFiles = set()

    for fiz in filesInZip:
        extractStats[fiz] = ''
        data = zf.read(fiz)
        new_hash = getGitBlobHash(data)
        if fiz in byName:
            curVer = byName[fiz]
            oldHash = curVer.key().parent().name()

            if oldHash == new_hash:
                extractStats[fiz] += 'Same version already the head version in the system. '
                continue
            else:
                extractStats[fiz] += 'Different version in zip file and system head. '
                if fiz not in forceFiles: continue

        theDataKey = datamodel.DB_FileContent.all(keys_only=True).filter('__key__ =', db.Key.from_path(u'DB_FileContent', new_hash)).get()
        if not theDataKey:
            theDataKey = datamodel.DB_FileContent(key_name = new_hash, data = db.Blob(data)).put()
            extractStats[fiz] += 'Added to cache. '
        else:
            extractStats[fiz] += 'Already in cache. '
        
        nextVersionNum = datamodel.DB_JV_AtomicCounter.GetNextCounter(_fileVerKey)
        tags = ['z', 'a'] # make it head
        newFileVer = datamodel.DB_FileVersion(parent=theDataKey, key_name=str(nextVersionNum),
                                                    filename=fiz,version=nextVersionNum,uid=user.key().id(),tags=tags)
        newFileVer.put()
        extractStats[fiz] += 'Added version %s with hash: %s ' % (nextVersionNum, new_hash)
    #forceCacheRebuild(fiz, 'z')

    zf.close()
    return RetType.JSONSUCCESS, {'stats':extractStats}
Example #7
0
def admin_CreateAllFileZip(req):
    import cStringIO
    import zipfile
    zipstream = cStringIO.StringIO()
    zf = zipfile.ZipFile(zipstream, 'w', zipfile.ZIP_DEFLATED)

    allFiles = getAllFromFromQuery(datamodel.DB_FileVersion.all())

    def cleanDate(dd): return str(dd).replace(':', '.').replace(' ', '-')

    for x in allFiles:
        content = x.parent().data
        aa = x.filename.rsplit('.', 1)
        aa.insert(1, '%s_%s%s' % (str(x.version).zfill(4), cleanDate(x.dateAdded), '_a' if 'a' in x.tags else '') )
        tfn = '.'.join(aa)
        zf.writestr(str(tfn), content)

    zf.close()

    outfile = 'onlinefiles-' + cleanDate(datetime.datetime.now()) + '.zip'

    res = zipstream.getvalue()
    return RetType.HEADERSANDRAW, {'Content-Type':'application/zip', 'Content-Disposition': 'attachment; filename="'+outfile+'"' }, res