def import_manifest(arch,version):
v = Version.get_by_key_name(version)
if v is None or not v.imported:
m = fetcher.fetch(arch,version,'manifest.xml')
if m is not None:
m = Manifest(m)
#xg_on = db.create_transaction_options(xg=True)
v = version_ok(arch,version)
prev = db.GqlQuery('select * from Version where imported = True and arch = {0}'.format(arch)).fetch(1)
if prev is not None and len(prev) > 0:
prev = prev[0]
from htmldiff import Changeset
pmanifest = Manifest(fetcher.fetch(arch,prev.value,'manifest.xml'))
changes = Changeset(pmanifest,m)
to_delete = [ pmanifest.files[x] for x in changes.dels | changes.changes
if pmanifest.files[x]['path'].endswith('entity') and pmanifest.files[x]['path'].startswith('game/resources0.s2z')]
to_import = [ m.files[x] for x in changes.adds | changes.changes if
m.files[x]['path'].endswith('entity') and m.files[x]['path'].startswith('game/resources0.s2z')]
total = len(to_delete)
current = 1
del(changes)
del(m)
del(pmanifest)
for file in to_delete:
e = Node.get_by_key_name('|'.join([file['version'],file['path']]))
if e is not None:
logging.info('[{1}/{2}] Deleting {0} entity group'.format('|'.join([file['version'],file['path']]),current,total))
db.run_in_transaction(delete_group,e)
current += 1
del(to_delete)
else:
prev = None
to_import = [x for x in m.files.values() if x['path'].endswith('entity') and x['path'].startswith('game/resources0.s2z')]
total = len(to_import)
current = 1
for file in to_import:
if file['path'].endswith('.entity'):
e = Node.get_by_key_name('|'.join([file['version'],file['path']]))
if e is None:
data = fetcher.fetch(arch,file['version'],file['path'])
#if data is None:
#continue
logging.info('[%d/%d] importing %s %s into db' % (current,total,file['version'],file['path']))
db.run_in_transaction(parse_entity,data,file['version'],file['path'],[version])
#db.run_in_transaction_options(xg_on,parse_entity,file['version'],file['path'],[version])
#elif version not in e.versions:
#db.run_in_transaction(set_version,e,version)
current += 1
v.imported = True
v.put()
if prev is not None:
prev.imported = False
prev.put()
def get_page(self,arch,version):
error = ''
query = ''
keywords = ''
stored_key = self.request.get('stored_query')
if stored_key:
try:
stored = StoredQuery.get(stored_key)
if not stored:
error = 'Sorry, there was no such stored query'
else:
query = stored.query
keywords = stored.keywords
except:
error = 'Sorry, there was no such stored query'
else:
query = self.request.get('query')
query = urllib.unquote(query)
keywords = self.request.get('keywords')
template_values = {
'query' : query,
'version' : version,
}
query = query.strip()
data = []
v = Version.get_by_key_name(version)
if arch != ARCHS.LINUX_RETAIL:
return '<pre>Sorry, DB is disabled for RCT/SBT</pre>'
if v is None or not v.imported:
versions = get_versions(arch)
versions.sort(key = lambda x: [int(y) for y in x.split('.')])
if version == versions[-1]:
error = "Sorry, this version is not imported into db yet, importing was put into queue"
taskqueue.add(url='/import',params={'version' : version,'arch' : arch},queue_name='importer')
else:
self.redirect('/query/latest/?' + self.request.query_string)
else:
if len(query) > 0:
for qline in query.splitlines():
operation = ''
if qline[0] == '&':
operation = '&'
qline = qline[1:]
elif qline[0] == '!':
operation = '!'
qline = qline[1:]
try:
qline = "Select * from Node where {0}".format(qline)
logging.info(qline)
q = db.GqlQuery(qline)
except:
error = 'Sorry this query was malformed'
q = None
if q is not None:
pb = q._proto_query
inequalities = {}
for k,v in pb.filters().iteritems():
#keywords.append(k[0])
if k[1] == '!=':
inequalities[k[0]] = set([v[1][0]._Literal__value for v in v])
#logging.info(inequalities)
#for k in pb.orderings():
#keywords.append(k[0])
if len(pb.orderings()) > 0:
error = 'Sorry order by is not allowed'
else:
try:
result = q.fetch(1000)
#logging.info('results: {0}'.format(len(result)))
_result = []
for r in result:
ok = True
for prop,keys in inequalities.iteritems():
l = getattr(r,prop)
if isinstance(l,list):
l = set(l)
else:
l = set([l])
if len(l & keys) > 0:
ok = False
#logging.info('filtered')
#logging.info(l)
break
if not ok: break
if ok:
_result.append(r)
result = _result
_data = []
for node in result:
root = node
while root.parent() is not None:
root = root.parent()
_data.append((root.key().name(),root,node))
if operation == '&':
roots = set([x[0] for x in _data])
_data = [x for x in data if x[0] in roots]
data = _data
elif operation == '!':
roots = [x[0] for x in _data]
_data = [x for x in data if x[0] not in roots]
data = _data
else:
data.extend(_data)
except datastore_errors.NeedIndexError, exc:
x = str(exc)
error = 'Sorry, this query is not possible without additional indices'
def get_page(self,arch,version,hero):
v = Version.get_by_key_name(version)
if arch != fetcher.ARCHS.LINUX_RETAIL:
return '<pre>Sorry, DB is disabled for RCT/SBT</pre>'
elif v is None or not v.imported:
versions = get_versions()
versions.sort(key = lambda x: [int(y) for y in x.split('.')])
if version == versions[-1]:
self.response.out.write("Sorry, this version is not imported into db yet, importing was put into queue")
taskqueue.add(url='/import',params={'version' : version,'arch' : arch},queue_name='importer')
else:
self.redirect('/heroes/latest/?' + self.request.query_string)
return None
else:
if hero is None:
manifest = fetcher.fetch(arch,version,'manifest.xml')
manifest = Manifest(manifest)
query = "Select * from Node where tag='hero'".format(version)
q = db.GqlQuery(query)
result = q.fetch(1000)
result = [_ for _ in result if _.name not in ['wl_Warlock']]
for hero in result:
if hasattr(hero,'attackprojectile') and hero.attackprojectile != '':
projectile = db.GqlQuery("Select * from Node where name='{0}'".format(hero.attackprojectile)).fetch(1)[0]
if hasattr(projectile,'speed'):
hero.projectilespeed = projectile.speed
else:
hero.projectilespeed = '""'
else:
hero.projectilespeed = '""'
#get url for icon
icon = hero.icon.replace('.tga','.dds')
path = '/'.join(hero.key().name().split('|')[1].split('/')[:-1])
path = '/'.join([path,icon])
path = path.replace('game/resources0.s2z','game/textures.s2z/00000000')
if path in manifest.files:
path = '/'.join([manifest.files[path]['version'],path])
else:
logging.info("Failed to create url for hero icon :( :")
logging.info(icon)
logging.info(path)
hero.iconurl = path
template_values = {}
template_values['data'] = result
template_values['stringtables'] = get_stringtables_entities(arch,version)
template_name = self.request.get('template')
if template_name and template_name == 'csv':
template = templates.get_template('heroes.csv')
else:
template = templates.get_template('heroes.html')
#self.response.out.write(template.render(template_values))
#return None
return template.render(template_values)
else:
hero = db.GqlQuery("Select * from Node where tag='hero' and name = :1",hero).fetch(1)
if len(hero) == 0:
return 'Sorry, such hero is not found'
hero = hero[0]
#get url for icon
manifest = fetcher.fetch(arch,version,'manifest.xml')
manifest = Manifest(manifest)
icon = hero.icon.replace('.tga','.dds')
path = '/'.join(hero.key().name().split('|')[1].split('/')[:-1])
path = '/'.join([path,icon])
path = path.replace('game/resources0.s2z','game/textures.s2z/00000000')
path = '/'.join([manifest.files[path]['version'],path])
hero.iconurl = path
abilities = db.GqlQuery("Select * from Node where tag='ability' and name in :1",[hero.inventory0,hero.inventory1,hero.inventory2,hero.inventory3]).fetch(10)
for a in abilities:
icon = a.icon.replace('.tga','.dds')
path = '/'.join(a.key().name().split('|')[1].split('/')[:-1])
path = '/'.join([path,icon])
path = path.replace('game/resources0.s2z','game/textures.s2z/00000000')
path = '/'.join([manifest.files[path]['version'],path])
a.iconurl = path
#abilities = dict([(a.name,a) for a in abilities])
template_values = {}
template_values['entity'] = hero
template_values['version'] = version
template_values['abilities'] = abilities
template_values['stringtables'] = get_stringtables_entities(arch,version)
template = templates.get_template('hero.html')
return template.render(template_values)