Exemple #1
0
def import_manifest(arch,version):
    v = Version.get_by_key_name(version)
    if v is None or not v.imported:
        m = fetcher.fetch(arch,version,'manifest.xml')
        if m is not None:
            m = Manifest(m)
            #xg_on = db.create_transaction_options(xg=True)
            v = version_ok(arch,version)
            prev = db.GqlQuery('select * from Version where imported = True and arch = {0}'.format(arch)).fetch(1)
            if prev is not None and len(prev) > 0:
                prev = prev[0]
                from htmldiff import Changeset
                pmanifest = Manifest(fetcher.fetch(arch,prev.value,'manifest.xml'))
                changes = Changeset(pmanifest,m)
                to_delete = [ pmanifest.files[x] for x in changes.dels | changes.changes 
                        if pmanifest.files[x]['path'].endswith('entity') and pmanifest.files[x]['path'].startswith('game/resources0.s2z')]
                to_import = [ m.files[x] for x in changes.adds | changes.changes if 
                        m.files[x]['path'].endswith('entity') and m.files[x]['path'].startswith('game/resources0.s2z')]
                total = len(to_delete)
                current = 1
                del(changes)
                del(m)
                del(pmanifest)
                for file in to_delete:
                    e = Node.get_by_key_name('|'.join([file['version'],file['path']]))
                    if e is not None:
                        logging.info('[{1}/{2}] Deleting {0} entity group'.format('|'.join([file['version'],file['path']]),current,total))
                        db.run_in_transaction(delete_group,e)
                    current += 1
                del(to_delete)
            else:
                prev = None
                to_import = [x for x in m.files.values() if x['path'].endswith('entity') and x['path'].startswith('game/resources0.s2z')]
            
            total = len(to_import)
            current = 1
            for file in to_import:
                if file['path'].endswith('.entity'):
                    e = Node.get_by_key_name('|'.join([file['version'],file['path']]))
                    if e is None:
                        data = fetcher.fetch(arch,file['version'],file['path'])
                        #if data is None:
                            #continue
                        logging.info('[%d/%d] importing %s %s into db' % (current,total,file['version'],file['path']))
                        db.run_in_transaction(parse_entity,data,file['version'],file['path'],[version])
                        #db.run_in_transaction_options(xg_on,parse_entity,file['version'],file['path'],[version])
                    #elif version not in e.versions:
                        #db.run_in_transaction(set_version,e,version)
                current += 1
            v.imported = True
            v.put()
            if prev is not None:
                prev.imported = False
                prev.put()
Exemple #2
0
    def get_page(self,arch,version):
        error = ''
        query = ''
        keywords = ''
        stored_key = self.request.get('stored_query')
        if stored_key:
            try:
                stored = StoredQuery.get(stored_key)
                if not stored:
                    error = 'Sorry, there was no such stored query'
                else:
                    query = stored.query
                    keywords = stored.keywords
            except:
                error = 'Sorry, there was no such stored query'
        else:
            query = self.request.get('query')
            query = urllib.unquote(query)
            keywords = self.request.get('keywords')

        template_values = {
            'query' : query,
            'version' : version,
            }

        query = query.strip()
        data = []

        v = Version.get_by_key_name(version)
        if arch != ARCHS.LINUX_RETAIL:
            return '<pre>Sorry, DB is disabled for RCT/SBT</pre>'
        if v is None or not v.imported:
            versions = get_versions(arch)
            versions.sort(key = lambda x: [int(y) for y in x.split('.')])
            if version == versions[-1]:
                error = "Sorry, this version is not imported into db yet, importing was put into queue"
                taskqueue.add(url='/import',params={'version' : version,'arch' : arch},queue_name='importer')
            else:
                self.redirect('/query/latest/?' + self.request.query_string)
        else:
            if len(query) > 0:
                for qline in query.splitlines():
                    operation = ''
                    if qline[0] == '&':
                        operation = '&'
                        qline = qline[1:]
                    elif qline[0] == '!':
                        operation = '!'
                        qline = qline[1:]
                    try:
                        qline = "Select * from Node where {0}".format(qline)
                        logging.info(qline)
                        q = db.GqlQuery(qline)
                    except:
                        error = 'Sorry this query was malformed'
                        q = None
                    if q is not None:
                        pb = q._proto_query
                        inequalities = {}
                        for k,v in pb.filters().iteritems():
                            #keywords.append(k[0])
                            if k[1] == '!=':
                                inequalities[k[0]] = set([v[1][0]._Literal__value for v in v])
                        #logging.info(inequalities)
                        #for k in pb.orderings():
                            #keywords.append(k[0])
                        if len(pb.orderings()) > 0:
                            error = 'Sorry order by is not allowed'
                        else:

                            try:
                                result = q.fetch(1000)
                                #logging.info('results: {0}'.format(len(result)))
                                _result = []
                                for r in result:
                                    ok = True
                                    for prop,keys in inequalities.iteritems():
                                        l = getattr(r,prop)
                                        if isinstance(l,list):
                                            l = set(l)
                                        else:
                                            l = set([l])
                                        if len(l & keys) > 0:
                                            ok = False
                                            #logging.info('filtered')
                                            #logging.info(l)
                                            break
                                        if not ok: break
                                    if ok:
                                        _result.append(r)
                                result = _result
                                _data = []
                                for node in result:
                                    root = node
                                    while root.parent() is not None:
                                        root = root.parent()
                                    _data.append((root.key().name(),root,node))
                                if operation == '&':
                                    roots = set([x[0] for x in _data])
                                    _data = [x for x in data if x[0] in roots]
                                    data = _data
                                elif operation == '!':
                                    roots = [x[0] for x in _data]
                                    _data = [x for x in data if x[0] not in roots]
                                    data = _data
                                else:
                                    data.extend(_data)

                            except datastore_errors.NeedIndexError, exc:
                                x = str(exc)
                                error = 'Sorry, this query is not possible without additional indices'
Exemple #3
0
    def get_page(self,arch,version,hero):
        v = Version.get_by_key_name(version)
        if arch != fetcher.ARCHS.LINUX_RETAIL:
            return '<pre>Sorry, DB is disabled for RCT/SBT</pre>'
        elif v is None or not v.imported:
            versions = get_versions()
            versions.sort(key = lambda x: [int(y) for y in x.split('.')])
            if version == versions[-1]:
                self.response.out.write("Sorry, this version is not imported into db yet, importing was put into queue")
                taskqueue.add(url='/import',params={'version' : version,'arch' : arch},queue_name='importer')
            else:
                self.redirect('/heroes/latest/?' + self.request.query_string)
            return None
        else:
            if hero is None:
                manifest = fetcher.fetch(arch,version,'manifest.xml')
                manifest = Manifest(manifest)
                query = "Select * from Node where tag='hero'".format(version)
                q = db.GqlQuery(query)
                result = q.fetch(1000)
                result = [_ for _ in result if _.name not in ['wl_Warlock']]
                for hero in result:
                    if hasattr(hero,'attackprojectile') and hero.attackprojectile != '':
                        projectile = db.GqlQuery("Select * from Node where name='{0}'".format(hero.attackprojectile)).fetch(1)[0]
                        if hasattr(projectile,'speed'):
                            hero.projectilespeed = projectile.speed
                        else:
                            hero.projectilespeed = '""'
                    else:
                        hero.projectilespeed = '""'

                    #get url for icon
                    icon = hero.icon.replace('.tga','.dds')
                    path = '/'.join(hero.key().name().split('|')[1].split('/')[:-1])
                    path = '/'.join([path,icon])
                    path = path.replace('game/resources0.s2z','game/textures.s2z/00000000')
                    if path in manifest.files:
                        path = '/'.join([manifest.files[path]['version'],path])
                    else:
                        logging.info("Failed to create url for hero icon :( :")
                        logging.info(icon)
                        logging.info(path)
                    hero.iconurl = path
                template_values = {}
                template_values['data'] = result
                template_values['stringtables'] = get_stringtables_entities(arch,version)
                template_name = self.request.get('template')
                if template_name and template_name == 'csv':
                    template = templates.get_template('heroes.csv')        
                else:
                    template = templates.get_template('heroes.html')        
                #self.response.out.write(template.render(template_values))
                #return None
                return template.render(template_values)
            else:
                hero = db.GqlQuery("Select * from Node where tag='hero' and name = :1",hero).fetch(1)
                if len(hero) == 0:
                    return 'Sorry, such hero is not found'
                hero = hero[0]
                #get url for icon
                manifest = fetcher.fetch(arch,version,'manifest.xml')
                manifest = Manifest(manifest)
                icon = hero.icon.replace('.tga','.dds')
                path = '/'.join(hero.key().name().split('|')[1].split('/')[:-1])
                path = '/'.join([path,icon])
                path = path.replace('game/resources0.s2z','game/textures.s2z/00000000')
                path = '/'.join([manifest.files[path]['version'],path])
                hero.iconurl = path
                abilities = db.GqlQuery("Select * from Node where tag='ability' and name in :1",[hero.inventory0,hero.inventory1,hero.inventory2,hero.inventory3]).fetch(10)
                for a in abilities:
                    icon = a.icon.replace('.tga','.dds')
                    path = '/'.join(a.key().name().split('|')[1].split('/')[:-1])
                    path = '/'.join([path,icon])
                    path = path.replace('game/resources0.s2z','game/textures.s2z/00000000')
                    path = '/'.join([manifest.files[path]['version'],path])
                    a.iconurl = path

                #abilities = dict([(a.name,a) for a in abilities])
                template_values = {}
                template_values['entity'] = hero
                template_values['version'] = version
                template_values['abilities'] = abilities
                template_values['stringtables'] = get_stringtables_entities(arch,version)
                template = templates.get_template('hero.html')        
                return template.render(template_values)