def main():
cacher = XNovaPageCache()
cacher.save_load_encoding = 'UTF-8' # force encoding
cacher.load_from_disk_cache(clean=True)
#
# parse overview
content = cacher.get_page('overview')
p1 = OverviewParser()
p1.parse_page_content(content)
server_time = p1.server_time
logger.info('Flights follow:')
for fl in p1.flights:
logger.info(fl)
#
# parse user info
content = cacher.get_page('self_user_info')
p2 = UserInfoParser()
p2.parse_page_content(content)
#
# parse imperium
p3 = ImperiumParser()
content = cacher.get_page('imperium')
p3.parse_page_content(content)
#
# current planet
p4 = CurPlanetParser()
content = cacher.get_page(
'overview') # can be almost any page, overview or imperium is fine
p4.parse_page_content(content)
#
# galaxy
gp = GalaxyParser()
content = cacher.get_page('galaxy_1_7')
gp.parse_page_content(content)
if gp.script_body != '':
gp.unscramble_galaxy_script()
logger.info('Galaxy rows follow:')
logger.info(gp.galaxy_rows)
#
# planet buildings
pbp = PlanetBuildingsProgressParser()
pba = PlanetBuildingsAvailParser()
content = cacher.get_page('buildings_57064') # Tama
pba.parse_page_content(content)
pbp.parse_page_content(content)
logger.info('Planet buildings follow:')
for ba in pba.builds_avail:
logger.info(str(ba))
logger.info('Planet builds in progress follow:')
for bp in pbp.builds_in_progress:
logger.info(str(bp))
#
# planet shipyard
content = cacher.get_page('shipyard_57064') # Tama's shipyard
psyp = ShipyardBuildsInProgressParser()
psyp.server_time = server_time
psyp.parse_page_content(content)
# planet ships available
psap = ShipyardShipsAvailParser()
psap.parse_page_content(content)
logger.info('Planet ships available follow:')
for sa in psap.ships_avail:
logger.info(str(sa))
# planet defenses
psyp.clear()
psyp.server_time = server_time
psyp.parse_page_content(
cacher.get_page('defense_57064')) # Tama's defenses
psap.clear()
psap.parse_page_content(
cacher.get_page('defense_57064')) # Tama's defenses
logger.info('Planet defense available follow:')
for sa in psap.ships_avail:
logger.info(str(sa))
#
# planet researches available
content = cacher.get_page('research_57064') # Tama
prap = ResearchAvailParser()
prap.server_time = server_time
prap.parse_page_content(content)
logger.info('Planet researches available follow:')
for ra in prap.researches_avail:
logger.info(str(ra))
# planet factory researches
content = cacher.get_page('researchfleet_57064') # Tama
prap.clear()
prap.server_time = server_time
prap.set_parsing_research_fleet(True)
prap.parse_page_content(content)
logger.info('Planet factory researches available follow:')
for ra in prap.researches_avail:
logger.info(str(ra))
#
# techtree
ttp = TechtreeParser()
content = cacher.get_page('techtree')
ttp.parse_page_content(content)
#
# planet energy parser test
pep = PlanetEnergyResParser()
content = cacher.get_page(
'buildings_82160') # can be any overview, research or shipyard page
pep.parse_page_content(content)
logger.info('Parsed planet energy status: {0}/{1}'.format(
pep.energy_left, pep.energy_total))
#
# fllets counter
fmp = FleetsMaxParser()
content = cacher.get_page('fleet')
fmp.parse_page_content(content)
logger.info('Fleets: {0}/{1}, Expeditions: {2}/{3}'.format(
fmp.fleets_cur, fmp.fleets_max, fmp.expeditions_cur,
fmp.expeditions_max))
Chrome/45.0.2454.85 Safari/537.36'
delay_between_requests_secs = 5
# galaxy_range = (5, 5) # debug, originally (1, 5)
# system_range = (75, 75) # debug, originally (1, 499)
galaxy_range = (1, 5)
system_range = (1, 499)
max_cache_secs = 10 * 3600 # cache galaxy pages for 10 hours
status_filename = 'galaxy_auto_parser.json'
###############################################
# internal state vars, do not touch
logger = xn_logger.get('GAP', debug=True)
g_page_cache = XNovaPageCache()
g_page_dnl = XNovaPageDownload()
g_parser = GalaxyParser()
g_db = sqlite3.connect('galaxy.db')
g_got_from_cache = False
def int_(val):
if val is None:
return None
try:
r = int(val)
except ValueError:
r = 0
return r
def str_(val):
def main():
cacher = XNovaPageCache()
cacher.save_load_encoding = 'UTF-8' # force encoding
cacher.load_from_disk_cache(clean=True)
#
# parse overview
content = cacher.get_page('overview')
p1 = OverviewParser()
p1.parse_page_content(content)
server_time = p1.server_time
logger.info('Flights follow:')
for fl in p1.flights:
logger.info(fl)
#
# parse user info
content = cacher.get_page('self_user_info')
p2 = UserInfoParser()
p2.parse_page_content(content)
#
# parse imperium
p3 = ImperiumParser()
content = cacher.get_page('imperium')
p3.parse_page_content(content)
#
# current planet
p4 = CurPlanetParser()
content = cacher.get_page('overview') # can be almost any page, overview or imperium is fine
p4.parse_page_content(content)
#
# galaxy
gp = GalaxyParser()
content = cacher.get_page('galaxy_1_7')
gp.parse_page_content(content)
if gp.script_body != '':
gp.unscramble_galaxy_script()
logger.info('Galaxy rows follow:')
logger.info(gp.galaxy_rows)
#
# planet buildings
pbp = PlanetBuildingsProgressParser()
pba = PlanetBuildingsAvailParser()
content = cacher.get_page('buildings_57064') # Tama
pba.parse_page_content(content)
pbp.parse_page_content(content)
logger.info('Planet buildings follow:')
for ba in pba.builds_avail:
logger.info(str(ba))
logger.info('Planet builds in progress follow:')
for bp in pbp.builds_in_progress:
logger.info(str(bp))
#
# planet shipyard
content = cacher.get_page('shipyard_57064') # Tama's shipyard
psyp = ShipyardBuildsInProgressParser()
psyp.server_time = server_time
psyp.parse_page_content(content)
# planet ships available
psap = ShipyardShipsAvailParser()
psap.parse_page_content(content)
logger.info('Planet ships available follow:')
for sa in psap.ships_avail:
logger.info(str(sa))
# planet defenses
psyp.clear()
psyp.server_time = server_time
psyp.parse_page_content(cacher.get_page('defense_57064')) # Tama's defenses
psap.clear()
psap.parse_page_content(cacher.get_page('defense_57064')) # Tama's defenses
logger.info('Planet defense available follow:')
for sa in psap.ships_avail:
logger.info(str(sa))
#
# planet researches available
content = cacher.get_page('research_57064') # Tama
prap = ResearchAvailParser()
prap.server_time = server_time
prap.parse_page_content(content)
logger.info('Planet researches available follow:')
for ra in prap.researches_avail:
logger.info(str(ra))
# planet factory researches
content = cacher.get_page('researchfleet_57064') # Tama
prap.clear()
prap.server_time = server_time
prap.set_parsing_research_fleet(True)
prap.parse_page_content(content)
logger.info('Planet factory researches available follow:')
for ra in prap.researches_avail:
logger.info(str(ra))
#
# techtree
ttp = TechtreeParser()
content = cacher.get_page('techtree')
ttp.parse_page_content(content)
#
# planet energy parser test
pep = PlanetEnergyResParser()
content = cacher.get_page('buildings_82160') # can be any overview, research or shipyard page
pep.parse_page_content(content)
logger.info('Parsed planet energy status: {0}/{1}'.format(pep.energy_left, pep.energy_total))
#
# fllets counter
fmp = FleetsMaxParser()
content = cacher.get_page('fleet')
fmp.parse_page_content(content)
logger.info('Fleets: {0}/{1}, Expeditions: {2}/{3}'.format(
fmp.fleets_cur, fmp.fleets_max, fmp.expeditions_cur, fmp.expeditions_max))
def test_parse_galaxy(self):
content = read_test_page('galaxy_5_62.html')
self.assertIsNotNone(content)
parser = GalaxyParser()
parser.parse_page_content(content)
#
galaxy_rows = parser.galaxy_rows
#
self.assertIsNone(galaxy_rows[0])
self.assertIsNone(galaxy_rows[1])
self.assertIsNone(galaxy_rows[2])
# check 3rd planet every field
self.assertIsNotNone(galaxy_rows[3])
self.assertIsInstance(galaxy_rows[3], dict)
self.assertEqual(galaxy_rows[3]['ally_planet'], 0)
self.assertIsNone(galaxy_rows[3]['luna_temp'])
self.assertEqual(galaxy_rows[3]['metal'], 0)
self.assertEqual(galaxy_rows[3]['id_planet'], 82160)
self.assertIsNone(galaxy_rows[3]['ally_web'])
self.assertIsNone(galaxy_rows[3]['type'])
self.assertEqual(galaxy_rows[3]['avatar'], 7)
self.assertEqual(galaxy_rows[3]['image'], 'trockenplanet18')
self.assertEqual(galaxy_rows[3]['onlinetime'], 0)
self.assertEqual(galaxy_rows[3]['username'], 'ElvenShadow')
self.assertEqual(galaxy_rows[3]['parent_planet'], 0)
self.assertEqual(galaxy_rows[3]['ally_id'], 0)
self.assertEqual(galaxy_rows[3]['name'], 'Андромеда')
self.assertIsNone(galaxy_rows[3]['luna_diameter'])
self.assertEqual(galaxy_rows[3]['crystal'], 0)
self.assertIsNone(galaxy_rows[3]['ally_members'])
self.assertEqual(galaxy_rows[3]['sex'], 1)
self.assertEqual(galaxy_rows[3]['planet_type'], 1)
self.assertEqual(galaxy_rows[3]['user_id'], 114604)
self.assertEqual(galaxy_rows[3]['race'], 2)
self.assertIsNone(galaxy_rows[3]['ally_name'])
self.assertEqual(galaxy_rows[3]['destruyed'], 0)
self.assertEqual(galaxy_rows[3]['total_points'], 3509)
self.assertIsNone(galaxy_rows[3]['luna_id'])
self.assertEqual(galaxy_rows[3]['last_active'], 55)
self.assertEqual(galaxy_rows[3]['planet'], 3)
self.assertEqual(galaxy_rows[3]['user_image'], '114604_1451805300.jpg')
self.assertEqual(galaxy_rows[3]['authlevel'], 0)
self.assertIsNone(galaxy_rows[3]['luna_destruyed'])
self.assertIsNone(galaxy_rows[3]['ally_tag'])
self.assertEqual(galaxy_rows[3]['banaday'], 0)
self.assertEqual(galaxy_rows[3]['total_rank'], 179)
self.assertIsNone(galaxy_rows[3]['luna_name'])
self.assertEqual(galaxy_rows[3]['urlaubs_modus_time'], 0)
# 5th planet is (i) player
self.assertEqual(galaxy_rows[5]['onlinetime'], 1)
# 6th planet is (U) player, has ally
self.assertEqual(galaxy_rows[6]['urlaubs_modus_time'], 1)
self.assertEqual(galaxy_rows[6]['ally_web'], '')
self.assertEqual(galaxy_rows[6]['ally_id'], 6)
self.assertEqual(galaxy_rows[6]['ally_members'], 7)
self.assertEqual(galaxy_rows[6]['ally_name'], \
'Великолепный МИФ или мифитерия Ж')
self.assertEqual(galaxy_rows[6]['ally_tag'], 'МиФ')
# 8th planet has moon
self.assertEqual(galaxy_rows[8]['luna_diameter'], 8660)
self.assertEqual(galaxy_rows[8]['luna_id'], 55144)
self.assertEqual(galaxy_rows[8]['luna_destruyed'], 0)
self.assertEqual(galaxy_rows[8]['luna_name'], 'Мамба')
self.assertEqual(galaxy_rows[8]['luna_temp'], 47)