def get_arena_teams(self, battlegroup, realm, site, get_characters=False, \
ladders=[2,3,5], max_pages=None):
'''Returns a list of arena teams as team objects. Setting get_characters
to true will cause teams, their characters and their guilds to be
downloaded. This cascading effect is very slow and should be used
with caution.
'''
all_teams = []
for ladder_number in ladders:
try:
source = self._download_url( \
WoWSpyderLib.get_arena_url(battlegroup, realm, site, \
ladder_number=ladder_number))
except Exception, e:
log.warning("Couldn't get arena page for ladder " +
str(ladder_number) + ", continuing. ERROR: " + str(e))
continue
if not max_pages:
try:
max_pages = WoWSpyderLib.get_max_pages(source)
except AttributeError, e:
# cflewis | 2009-04-22 | This means that
# this arena will be skipped, but it's better than
# crashing
max_pages = 0
def _get_character_achievements(self, name, realm, site):
urls = WoWSpyderLib.get_character_achievement_urls(name, realm, site)
achievements = []
for url in urls:
source = self._download_url(url)
achievements.append(self._parse_character_achievements( \
StringIO.StringIO(source), name, realm, site))
return WoWSpyderLib.merge(achievements)
def _get_character_statistics(self, name, realm, site):
urls = WoWSpyderLib.get_character_statistics_urls(name, realm, site)
statistics = []
for url in urls:
source = self._download_url(url)
statistics.append(self._parse_character_statistics( \
StringIO.StringIO(source), name, realm, site))
return WoWSpyderLib.merge(statistics)
def _parse_guild_characters(self, name, realm, site):
"""Page through a guild, creating characters."""
character_list = []
log.debug(name + " " + realm + ": Downloading guild page")
source = self._download_url( \
WoWSpyderLib.get_guild_url(name, realm, site, page=1))
character_list.append(self._parse_guild_file(StringIO.StringIO(source), site))
return WoWSpyderLib.merge(character_list)
def get_team(self, name, realm, site, size=None, get_characters=False, cached=False):
"""Returns a team object. Setting get_characters to True will
cause characters in the team to be created at the same time. This is
slower, but likely what you will want.
Setting cached to True will return the cached version of the team,
if you're sure it's already in the database.
"""
team = self._session.query(Team).get((name, realm, site))
if cached and team:
return team
if not team and not size: raise NameError("No team on that PK, " + \
"need size to create new team.")
log.debug("Getting team...")
# cflewis | 2009-04-02 | If downloading fails, the whole team
# couldn't be found, so the exception should propagate up.
source = self._download_url(\
WoWSpyderLib.get_team_url(name, realm, site, size))
team = self._parse_team(StringIO.StringIO(source), site, get_characters=get_characters)
return team
def get_guild_rank(self, guild_name, realm, site, character_name):
"""Returns the rank of a character (not specified by object)
in a guild.
"""
source = self._download_url( \
WoWSpyderLib.get_guild_url(guild_name, realm, site, page=1))
guild_rank_search = re.search("name=\"" + character_name + \
"\".*rank=\"(\d*)\"", unicode(source, "utf-8"))
if guild_rank_search:
return int(guild_rank_search.group(1))
raise IOError("No character in that guild")
def get_character(self, name, realm, site, cached=False, force_refresh=False):
"""Return a character object. This only stubs the guild, which means
the guild won't be populated with characters."""
log.debug("Getting character " + name + "...")
character = self._session.query(Character).get((name, realm, site))
# cflewis | 2009-04-11 | Check if a character is actually updated
# on the armory. If not, return the database version anyway.
if character and (cached == True or character.is_updated_on_armory() == False) and not force_refresh:
return character
source = self._download_url(\
WoWSpyderLib.get_character_sheet_url(name, realm, site))
character = self._parse_character(StringIO.StringIO(source), site)
return character
def get_guild(self, name, realm, site, get_characters=False, cached=False):
"""Get a guild. Setting get_characters=False will disable the
behavior that causes the guild characters to also be created. You
may want to do this for speed increases.
"""
if name is None or name == "":
return None
guild = self._session.query(Guild).get((name, realm, site))
if cached and guild:
return guild
# cflewis | 2009-04-02 | If the downloading fails, the whole guild
# couldn't be found, so the exception should propagate up.
source = self._download_url(\
WoWSpyderLib.get_guild_url(name, realm, site))
guild = self._parse_guild(StringIO.StringIO(source), site, get_characters=get_characters)
return guild
def last_modified_on_armory(self):
# cflewis | 2009-04-11 | It's safe to cache because there's no way
# the lifetime of a character object will exceed that of an armory
# refresh.
downloader = XMLDownloader.XMLDownloader()
source = downloader.download_url(self.url)
try:
if self._last_modified_on_armory != None:
return self._last_modified_on_armory
except AttributeError:
try:
armory_date_string = re.search("lastModified=\"(.*?)\"", source).group(1)
except Exception, e:
log.debug("Couldn't find last modified, returning what I had")
return self.last_modified
else:
self._last_modified_on_armory = \
WoWSpyderLib.convert_last_modified_to_datetime(armory_date_string)
log.debug("Saved last modified on armory as " + str(self._last_modified_on_armory))
return self._last_modified_on_armory
def get_item(self, item_id, cached=True):
"""Returns an item object.
Setting cached to True will return the cached version of the item,
if you're sure it's already in the database. This will fall through
if the item wasn't found. Because items are immutable, caching
is on by default.
"""
item = self._session.query(Item).get(item_id)
if cached and item:
return item
log.debug("Getting item...")
# cflewis | 2009-04-02 | If downloading fails, the whole team
# couldn't be found, so the exception should propagate up.
source = self._download_url(WoWSpyderLib.get_item_url(item_id))
item = self._parse_item(StringIO.StringIO(source))
return item
def url(self):
return WoWSpyderLib.get_team_url(self.name, self.realm, self.site, \
self.size)
def testGetEUArenaURLNonUnicode(self):
eu_url = WoWSpyderLib.get_arena_url(self.eu_battlegroup, self.eu_realm, "eu")
self.assertTrue(re.match("http://eu.wowarmory", eu_url))
else:
guild_name = guild.name
# cflewis | 2009-04-02 | Last modified continues to be weird and
# I can't track this bug down!
last_modified = None
log.debug("Guilds done, working on last modified date...")
try:
last_modified_string = character_node.attributes["lastModified"].value
except KeyError, e:
log.warning("Couldn't get last modified date. ERROR: " + str(e))
last_modified = None
else:
last_modified = WoWSpyderLib.convert_last_modified_to_datetime(last_modified_string)
log.debug("Last modified date is " + str(last_modified))
if last_modified.year < 2008:
log.warning("Last modified year was broken, fixing it to this year")
# cflewis | 2009-03-30 | The Armory has been returning strange
# years intermittently, not replicable when I manually visit
# the page. I'll set the year to what the current year is.
last_modified.replace(year=datetime.datetime.now().year)
else:
log.debug("Last modified year is " + str(last_modified.year) + " so continuing")
log.debug("Character done, getting items...")
items = []
for i in range(0, 19):
def testGetUSArenaURL(self):
us_url = WoWSpyderLib.get_arena_url(self.us_battlegroup, \
self.us_realm, u"us")
self.assertTrue(re.match("http://www.wowarmory", us_url))
log.debug(battlegroup + " " + realm + \
": Downloading arena page " + str(page) + " of " \
+ str(max_pages))
try:
source = self._download_url( \
WoWSpyderLib.get_arena_url(battlegroup, realm, site, page=page, \
ladder_number=ladder_number))
except Exception, e:
log.warning("Couldn't get arena page, continuing... ERROR: " + str(e))
continue
teams = self._parse_arena_file(StringIO.StringIO(source), site, get_characters=get_characters)
all_teams.append(teams)
return WoWSpyderLib.merge(all_teams)
def _parse_arena_file(self, xml_file_object, site, get_characters=False):
"""Parse the XML of an arena page"""
xml = minidom.parse(xml_file_object)
team_nodes = xml.getElementsByTagName("arenaTeam")
teams = []
for team_node in team_nodes:
name = team_node.attributes["name"].value
realm = team_node.attributes["realm"].value
size = team_node.attributes["size"].value
try:
team = self._tp.get_team(name, realm, site, size, get_characters=get_characters)
except Exception, e:
def _get_character_talents(self, name, realm, site):
source = self._download_url(\
WoWSpyderLib.get_character_talents_url(name, realm, site))
talents = self._parse_character_talents(StringIO.StringIO(source))
return talents
def get_url_page(self, page_number):
log.debug("Returning URL for " + self.name + "," + self.realm + "," + self.site)
return WoWSpyderLib.get_guild_url(self.name, self.realm, self.site)
def url(self):
return WoWSpyderLib.get_character_sheet_url(self.name, self.realm, self.site)
