def parseAchievement(title, attributes, c, imageCandidates):
name = title
if 'actualname' in attributes and len(attributes['actualname']) > 0:
name = formatTitle(attributes['actualname'])
elif 'name' in attributes and len(attributes['name']) > 0:
name = formatTitle(attributes['name'])
description = None
if 'description' in attributes:
description = attributes['description']
spoiler = None
if 'spoiler' in attributes:
spoiler = attributes['spoiler']
grade = None
if 'grade' in attributes:
try: grade = int(attributes['grade'])
except: pass
points = None
if 'points' in attributes:
try: points = int(attributes['points'])
except: pass
references = []
if description != None:
description = sanitizeText(description, references)
if spoiler != None:
spoiler = sanitizeText(spoiler, references)
references = flatten(references)
c.execute('INSERT INTO Achievements (name, grade, points, description, spoiler, image, imagetype) VALUES (?,?,?,?,?,?,?)',
(name, grade, points, description, spoiler, None, None))
achievementid = c.lastrowid
imageCandidates[achievementid] = references
return(True)
def parseAchievement(title, attributes, c, imageCandidates):
name = title
if 'actualname' in attributes and len(attributes['actualname']) > 0:
name = formatTitle(attributes['actualname'])
elif 'name' in attributes and len(attributes['name']) > 0:
name = formatTitle(attributes['name'])
description = None
if 'description' in attributes:
description = attributes['description']
spoiler = None
if 'spoiler' in attributes:
spoiler = attributes['spoiler']
grade = None
if 'grade' in attributes:
try:
grade = int(attributes['grade'])
except:
pass
points = None
if 'points' in attributes:
try:
points = int(attributes['points'])
except:
pass
references = []
if description != None:
description = sanitizeText(description, references)
if spoiler != None:
spoiler = sanitizeText(spoiler, references)
references = flatten(references)
c.execute(
'INSERT INTO Achievements (name, grade, points, description, spoiler, image, imagetype) VALUES (?,?,?,?,?,?,?)',
(name, grade, points, description, spoiler, None, None))
achievementid = c.lastrowid
imageCandidates[achievementid] = references
return (True)
def parseQuest(title, attributes, c, rewardItems, questDangers, getURL):
name = title
if 'name' in attributes:
name = formatTitle(attributes['name'])
lvl = 0
if 'lvl' in attributes:
try:
lvl = int(numberRegex.search(attributes['lvl']).groups()[0])
except:
pass
premium = False
if 'premium' in attributes:
premium = attributes['premium'].strip().lower() == 'yes'
legend = None
if 'legend' in attributes and attributes['legend'] != None:
legend = attributes['legend'].replace('[', '').replace(
']', '').replace('"', '"').replace('&', '&')
city = None
if 'location' in attributes:
loclow = attributes['location'].lower()
for candidatecity in cities:
if candidatecity in loclow:
city = candidatecity
break
if legend == None and city == None:
return False
c.execute(
'INSERT INTO Quests (title, name, minlevel, premium, city, legend) VALUES (?,?,?,?,?,?)',
(title, name, lvl, premium, city, legend))
questid = c.lastrowid
if 'reward' in attributes:
index = 0
rewardItems[questid] = list()
while True:
match = bracketRegex.search(attributes['reward'][index:])
if match == None: break
rewardItems[questid].append(match.groups()[0])
index += match.end()
if 'dangers' in attributes:
index = 0
questDangers[questid] = list()
while True:
match = bracketRegex.search(attributes['dangers'][index:])
if match == None: break
questDangers[questid].append(match.groups()[0])
index += match.end()
return True
def parseQuest(title, attributes, c, rewardItems, questDangers, getURL):
name = title
if 'name' in attributes:
name = formatTitle(attributes['name'])
lvl = 0
if 'lvl' in attributes:
try: lvl = int(numberRegex.search(attributes['lvl']).groups()[0])
except: pass
premium = False
if 'premium' in attributes:
premium = attributes['premium'].strip().lower() == 'yes'
legend = None
if 'legend' in attributes and attributes['legend'] != None:
legend = attributes['legend'].replace('[', '').replace(']', '').replace('"','"').replace('&','&')
city = None
if 'location' in attributes:
loclow = attributes['location'].lower()
for candidatecity in cities:
if candidatecity in loclow:
city = candidatecity
break
if legend == None and city == None:
return False
c.execute('INSERT INTO Quests (title, name, minlevel, premium, city, legend) VALUES (?,?,?,?,?,?)', (title, name, lvl, premium, city, legend))
questid = c.lastrowid
if 'reward' in attributes:
index = 0
rewardItems[questid] = list()
while True:
match = bracketRegex.search(attributes['reward'][index:])
if match == None: break
rewardItems[questid].append(match.groups()[0])
index += match.end()
if 'dangers' in attributes:
index = 0
questDangers[questid] = list()
while True:
match = bracketRegex.search(attributes['dangers'][index:])
if match == None: break
questDangers[questid].append(match.groups()[0])
index += match.end()
return True
def wordCount(input_string, word):
return sum(1 for _ in re.finditer(r'%s' % re.escape(word), input_string))
if not skipLoading:
for child in root.getchildren():
titleTag = child.find(getTag('title'))
if titleTag == None: continue
title = titleTag.text
if 'help:' in title.lower() or 'talk:' in title.lower(
) or 'template:' in title.lower() or 'updates/' in title.lower(
) or 'user:'******'loot/' in title.lower(
) or 'tibiawiki:' in title.lower():
continue
title = formatTitle(title)
revisionTag = child.find(getTag('revision'))
if revisionTag == None: continue
textTag = revisionTag.find(getTag('text'))
if textTag == None: continue
content = textTag.text
if content == None: continue
attributes = parseAttributes(content)
lcontent = content.lower()
if '/Spoiler' in title:
quest = title.replace('/Spoiler', '')
questNPCs[quest] = re.findall('\[\[([^]|]+)(?:|[^]]+)?\]\]',
lcontent)
elif wordCount(lcontent, 'infobox hunt|') == 1 or wordCount(
lcontent, 'infobox_hunt|') == 1:
#print('Hunt', title)
def parseCreature(title, attributes, c, creaturedrops, getURL):
if title in passList:
return False
name = title
if 'actualname' in attributes and len(attributes['actualname']) > 0:
name = formatTitle(attributes['actualname'])
elif 'name' in attributes and len(attributes['name']) > 0:
name = formatTitle(attributes['name'])
hp = getInteger(attributes, 'hp')
exp = None
if 'exp' in attributes:
try:
exp = int(attributes['exp'])
except:
pass
summon = None
if 'summon' in attributes:
try:
summon = int(attributes['summon'])
except:
pass
illusionable = getBoolean(attributes, 'illusionable')
pushable = getBoolean(attributes, 'pushable')
pushes = getBoolean(attributes, 'pushes')
paralysable = getBoolean(attributes, 'paraimmune')
senseinvis = getBoolean(attributes, 'senseinvis', True)
armor = getInteger(attributes, 'armor')
maxdmg = getMaxInteger(attributes, 'maxdmg')
physical = getInteger(attributes, 'physicalDmgMod')
holy = getInteger(attributes, 'holyDmgMod')
death = getInteger(attributes, 'deathDmgMod')
fire = getInteger(attributes, 'fireDmgMod')
energy = getInteger(attributes, 'energyDmgMod')
ice = getInteger(attributes, 'iceDmgMod')
earth = getInteger(attributes, 'earthDmgMod')
drown = getInteger(attributes, 'drownDmgMod')
lifedrain = getInteger(attributes, 'hpDrainDmgMod')
speed = getInteger(attributes, 'speed')
boss = False
if 'isboss' in attributes and attributes['isboss'].lower().strip(
) == 'yes':
boss = True
abilities = None
if 'abilities' in attributes:
# first take care of [[Fire Rune||Great Fireball]] => Great Fireball
b = re.sub(r'\[\[[^]|]+\|([^]]+)\]\]', '\g<1>',
attributes['abilities'])
# then take care of [[Fire Rune]] => Fire Rune
b = re.sub(r'\[\[([^]]+)\]\]', '\g<1>', b)
# sometimes there are links in single brackets [http:www.link.com] => remove htem
b = re.sub(r'\[[^]]+\]', '', b)
# if there are brackets without numbers, remove them (maybe not necessary)
b = re.sub(r'\(([^0-9]+)\)', '', b)
# replace double spaces with single spaces
b = b.replace(' ', ' ')
# if there are commas in brackets (300-500, Fire Damage) => replace the comma with a semicolon (for later splitting purposes)
abilities = re.sub(r'(\([^,)]+)\,([^)]+\))', '\g<1>;\g<2>', b)
url = "http://tibia.wikia.com/wiki/%s" % (title.replace(' ', '_'))
image = getImage(url, getURL, imageRegex, crop_image)
if image == None or image == False:
url = "http://tibia.wikia.com/wiki/File:%s.gif" % (title.replace(
' ', '_'))
image = getImage(url, getURL, imageRegex2, crop_image)
if image == None or image == False:
print('failed to get image for creature', title)
# add stuff to database
c.execute(
'INSERT INTO Creatures (title,name,health,experience,maxdamage,summon,illusionable,pushable,pushes,physical,holy,death,fire,energy,ice,earth,drown,lifedrain,paralysable,senseinvis,image,abilities,speed,armor,boss) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
(title, name, hp, exp, maxdmg, summon, illusionable, pushable, pushes,
physical, holy, death, fire, energy, ice, earth, drown, lifedrain,
paralysable, senseinvis, image, abilities, speed, armor, boss))
creatureid = c.lastrowid
creaturedrops[creatureid] = dict()
# for some reason loot statistics are not in the xml file, so we get it from the website
url = 'http://tibia.wikia.com/wiki/Loot_Statistics:%s' % title.replace(
' ', '_')
stats = getURL(url, True)
if stats != None:
loot_stats = list()
current_index = 0
while True:
match = lootListRegex.search(stats[current_index:])
if match == None: break
index = match.end()
match = lootListRegex.search(stats[current_index + index:])
if match == None: endindex = len(stats) - current_index
else: endindex = index + match.start()
kill_count = 0
match = re.search(
'([0-9]+) kills',
stats[current_index + index:current_index + endindex])
if match != None: kill_count = int(match.groups()[0])
list.append(
loot_stats,
[current_index + index, current_index + endindex, kill_count])
current_index = current_index + endindex
lootdrops = dict()
killcount = dict()
score = dict()
for i in range(len(loot_stats)):
index = loot_stats[i][0]
endindex = loot_stats[i][1]
kills = loot_stats[i][2]
lootdrops[i] = dict()
killcount[i] = kills
bag = False
highpercentage = False
while True:
match = wikiURLRegex.search(stats[index:endindex])
if match == None: break
item_name = filterItemName(match.groups()[0]).lower()
# creatures don't drop bags, but they used to in the past
# if there is a bag in the creature kills, we know it's old
startindex = index
index = index + match.end()
if index > endindex or item_name == "loot": break
match = lootCountRegex.search(stats[startindex:index])
if match == None:
mindrop = 1
maxdrop = 1
else:
mindrop = int(match.groups()[0])
maxdrop = int(match.groups()[1])
match = lootChanceRegex.search(stats[index:])
if match == None: break
percentage = float(match.groups()[0])
if percentage > 100:
highpercentage = True
index = index + match.end()
if item_name.strip() == "bag":
bag = True
else:
lootdrops[i][item_name] = (percentage, mindrop, maxdrop)
score[i] = 0 if bag else (
1 if highpercentage else len(lootdrops[i]))
maxdict = dict()
maxkey = ""
killcount[maxkey] = -1
score[maxkey] = -1
# pick the loot statistics with the most items
for key in lootdrops.keys():
if score[key] > score[maxkey]:
maxdict = lootdrops[key]
maxkey = key
elif score[key] == score[
maxkey] and killcount[key] > killcount[maxkey]:
# if the items are equal, pick the one with the highest kills
maxdict = lootdrops[key]
maxkey = key
creaturedrops[creatureid] = maxdict
# read the dropped items from the 'loot' attribute
if 'loot' in attributes:
lootItems = [
filterItemName(x).lower() for x in parseLoot(attributes['loot'])
]
for item in lootItems:
if item not in creaturedrops[creatureid]:
creaturedrops[creatureid][item] = (None, 1, 1)
return True
questNPCs = dict()
keyItems = dict()
durationCostMap = dict()
import re
def wordCount(input_string, word):
return sum(1 for _ in re.finditer(r'%s' % re.escape(word), input_string))
if not skipLoading:
for child in root.getchildren():
titleTag = child.find(getTag('title'))
if titleTag == None: continue
title = titleTag.text
if 'help:' in title.lower() or 'talk:' in title.lower() or 'template:' in title.lower() or 'updates/' in title.lower() or 'user:'******'loot/' in title.lower() or 'tibiawiki:' in title.lower(): continue
title = formatTitle(title)
revisionTag = child.find(getTag('revision'))
if revisionTag == None: continue
textTag = revisionTag.find(getTag('text'))
if textTag == None: continue
content = textTag.text
if content == None: continue
attributes = parseAttributes(content)
lcontent = content.lower()
if '/Spoiler' in title:
quest = title.replace('/Spoiler', '')
questNPCs[quest] = re.findall('\[\[([^]|]+)(?:|[^]]+)?\]\]', lcontent)
elif wordCount(lcontent, 'infobox hunt|') == 1 or wordCount(lcontent, 'infobox_hunt|') == 1:
#print('Hunt', title)
if not parseHunt(title, attributes, c, content, huntcreatures, getURL):
print('Hunt failed', title)
def parseCreature(title, attributes, c, creaturedrops, getURL):
if title in passList:
return False
name = title
if 'actualname' in attributes and len(attributes['actualname']) > 0:
name = formatTitle(attributes['actualname'])
elif 'name' in attributes and len(attributes['name']) > 0:
name = formatTitle(attributes['name'])
hp = getInteger(attributes, 'hp')
exp = None
if 'exp' in attributes:
try: exp = int(attributes['exp'])
except: pass
summon = None
if 'summon' in attributes:
try: summon = int(attributes['summon'])
except: pass
illusionable = getBoolean(attributes,'illusionable')
pushable = getBoolean(attributes,'pushable')
pushes = getBoolean(attributes,'pushes')
paralysable = getBoolean(attributes,'paraimmune')
senseinvis = getBoolean(attributes,'senseinvis', True)
armor = getInteger(attributes,'armor')
maxdmg = getMaxInteger(attributes,'maxdmg')
physical = getInteger(attributes,'physicalDmgMod')
holy = getInteger(attributes,'holyDmgMod')
death = getInteger(attributes,'deathDmgMod')
fire = getInteger(attributes,'fireDmgMod')
energy = getInteger(attributes,'energyDmgMod')
ice = getInteger(attributes,'iceDmgMod')
earth = getInteger(attributes,'earthDmgMod')
drown = getInteger(attributes,'drownDmgMod')
lifedrain = getInteger(attributes,'hpDrainDmgMod')
speed = getInteger(attributes,'speed')
boss = False
if 'isboss' in attributes and attributes['isboss'].lower().strip() == 'yes':
boss = True
abilities = None
if 'abilities' in attributes:
# first take care of [[Fire Rune||Great Fireball]] => Great Fireball
b = re.sub(r'\[\[[^]|]+\|([^]]+)\]\]', '\g<1>', attributes['abilities'])
# then take care of [[Fire Rune]] => Fire Rune
b = re.sub(r'\[\[([^]]+)\]\]', '\g<1>', b)
# sometimes there are links in single brackets [http:www.link.com] => remove htem
b = re.sub(r'\[[^]]+\]', '', b)
# if there are brackets without numbers, remove them (maybe not necessary)
b = re.sub(r'\(([^0-9]+)\)', '', b)
# replace double spaces with single spaces
b = b.replace(' ', ' ')
# if there are commas in brackets (300-500, Fire Damage) => replace the comma with a semicolon (for later splitting purposes)
abilities = re.sub(r'(\([^,)]+)\,([^)]+\))', '\g<1>;\g<2>', b)
url = "http://tibia.wikia.com/wiki/%s" % (title.replace(' ', '_'))
image = getImage(url, getURL, imageRegex, crop_image)
if image == None or image == False:
url = "http://tibia.wikia.com/wiki/File:%s.gif" % (title.replace(' ', '_'))
image = getImage(url, getURL, imageRegex2, crop_image)
if image == None or image == False:
print('failed to get image for creature', title)
return False
if physical == None:
print('pass', title)
# add stuff to database
c.execute('INSERT INTO Creatures (title,name,health,experience,maxdamage,summon,illusionable,pushable,pushes,physical,holy,death,fire,energy,ice,earth,drown,lifedrain,paralysable,senseinvis,image,abilities,speed,armor,boss) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
(title,name, hp, exp, maxdmg, summon, illusionable, pushable, pushes, physical, holy, death, fire, energy, ice, earth, drown, lifedrain, paralysable, senseinvis, image, abilities, speed, armor,boss))
creatureid = c.lastrowid
creaturedrops[creatureid] = dict()
# for some reason loot statistics are not in the xml file, so we get it from the website
url = 'http://tibia.wikia.com/wiki/Loot_Statistics:%s' % title.replace(' ','_')
stats = getURL(url, True)
if stats != None:
loot_stats = list()
current_index = 0
while True:
match = lootListRegex.search(stats[current_index:])
if match == None: break
index = match.end()
match = lootListRegex.search(stats[current_index + index:])
if match == None: endindex = len(stats) - current_index
else: endindex = index + match.start()
kill_count = 0
match = re.search('([0-9]+) kills', stats[current_index + index:current_index + endindex])
if match != None: kill_count = int(match.groups()[0])
list.append(loot_stats, [current_index + index, current_index + endindex, kill_count])
current_index = current_index + endindex
lootdrops = dict()
killcount = dict()
score = dict()
for i in range(len(loot_stats)):
index = loot_stats[i][0]
endindex = loot_stats[i][1]
kills = loot_stats[i][2]
lootdrops[i] = dict()
killcount[i] = kills
bag = False
highpercentage = False
while True:
match = wikiURLRegex.search(stats[index:endindex])
if match == None: break
item_name = filterItemName(match.groups()[0]).lower()
# creatures don't drop bags, but they used to in the past
# if there is a bag in the creature kills, we know it's old
startindex = index
index = index + match.end()
if index > endindex or item_name == "loot": break
match = lootCountRegex.search(stats[startindex:index])
if match == None:
mindrop = 1
maxdrop = 1
else:
mindrop = int(match.groups()[0])
maxdrop = int(match.groups()[1])
match = lootChanceRegex.search(stats[index:])
if match == None: break
percentage = float(match.groups()[0])
if percentage > 100:
highpercentage = True
index = index + match.end()
if item_name.strip() == "bag":
bag = True
else:
lootdrops[i][item_name] = (percentage, mindrop, maxdrop)
score[i] = 0 if bag else (1 if highpercentage else len(lootdrops[i]))
maxdict = dict()
maxkey = ""
killcount[maxkey] = -1
score[maxkey] = -1
# pick the loot statistics with the most items
for key in lootdrops.keys():
if score[key] > score[maxkey]:
maxdict = lootdrops[key]
maxkey = key
elif score[key] == score[maxkey] and killcount[key] > killcount[maxkey]:
# if the items are equal, pick the one with the highest kills
maxdict = lootdrops[key]
maxkey = key
creaturedrops[creatureid] = maxdict
# read the dropped items from the 'loot' attribute
if 'loot' in attributes:
lootItems = [filterItemName(x).lower() for x in parseLoot(attributes['loot'])]
for item in lootItems:
if item not in creaturedrops[creatureid]:
creaturedrops[creatureid][item] = (None, 1, 1)
return True