def buildchar(charname, skills, standings):
char = Character()
char.name = charname
char.total_sp = 0
char.save()
for skill in skills:
cs = CharSkill()
cs.character = char
if skill[0] in STUPID_OLDNAMELOOKUP:
cs.skill = Skill.objects.filter(name=STUPID_OLDNAMELOOKUP[skill[0]])[0]
else:
cs.skill = Skill.objects.filter(name=skill[0])[0]
cs.level = skill[1]
cs.skill_points = skill[2]
cs.save()
char.skills.add(cs)
char.total_sp += skill[2]
for standing in standings:
try:
corp = NPC_Corp.objects.get(name=standing[0])
except ObjectDoesNotExist:
corp = NPC_Corp.objects.create(name=standing[0])
print 'Created new npc corp', standing[0]
corp.save()
global NEW_CORPS
NEW_CORPS.append(corp)
char.standings.add(Standing.objects.create(corp=corp, value=standing[1]))
if standings:
for corp in NPC_Corp.objects.all():
try:
char.standings.get(corp=corp)
except ObjectDoesNotExist:
char.standings.add(Standing.objects.create(corp=corp, value=0))
char.save()
return char
def refresh_characters(limit, staleness):
session = requests.Session()
staledate = now() - timedelta(days=staleness)
update_limit = now() - timedelta(days=limit)
updated_threads = Thread.objects.filter(last_update__gte=update_limit, blacklisted=False)
stale_characters = Character.objects.filter(
Q(last_update__lte=staledate) | Q(last_update=None), thread__in=updated_threads)
for character in stale_characters:
logger.debug("Updating stale character %s" % character.name)
scraped_info = scrape_character(session, character.name, character.password)
if scraped_info:
new_sp_total = 0
for skill in scraped_info['skills']:
existing_skill = character.skills.filter(skill__name=skill[0])
if len(existing_skill) > 0:
existing_skill[0].skill_points = skill[2]
existing_skill[0].level = skill[1]
new_sp_total += skill[2]
existing_skill[0].save()
else:
cs = CharSkill()
cs.character = character
cs.level = skill[1]
cs.skill_points = skill[2]
cs.typeID = cs.skill.typeID
cs.save()
character.skills.add(cs)
new_sp_total += skill[2]
character.remaps = scraped_info['stats']['remaps']
character.unspent_skillpoints = scraped_info['stats']['unallocated_sp']
character.total_sp = new_sp_total
character.last_update = now()
character.save()
logger.debug("Update of stale character %s complete" % character.name)
def buildchar(char_dict):
logger.debug("Building new character, %s" % char_dict['charname'])
char = Character()
char.name = char_dict['charname']
char.total_sp = 0
char.save()
for skill in char_dict['skills']:
cs = CharSkill()
cs.character = char
if skill[0] in STUPID_OLDNAMELOOKUP:
cs.skill = Skill.objects.filter(
name=STUPID_OLDNAMELOOKUP[skill[0]])[0]
else:
cs.skill = Skill.objects.filter(name=skill[0])[0]
cs.level = skill[1]
cs.skill_points = skill[2]
cs.typeID = cs.skill.typeID
cs.save()
logger.debug("Created CharSkill for {0}".format(skill))
char.skills.add(cs)
char.total_sp += skill[2]
for standing in char_dict['standings']:
corp = NPC_Corp.objects.filter(name=standing[0]).first()
if corp:
char.standings.add(
Standing.objects.create(corp=corp, value=standing[1]))
else:
corp = NPC_Corp.objects.create(name=standing[0])
corp.save()
char.standings.add(
Standing.objects.create(corp=corp, value=standing[1]))
logger.info('Created new npc corp {0}'.format(standing[0]))
char.last_update = now()
char.unspent_skillpoints = char_dict['stats']['unallocated_sp']
char.remaps = char_dict['stats']['remaps']
char.password = char_dict['password'] or ''
char.save()
logger.debug("Character built {0}".format(char_dict['charname']))
return char
def buildchar(char_dict):
logger.debug("Building new character, %s" % char_dict['charname'])
char = Character()
char.name = char_dict['charname']
char.total_sp = 0
char.save()
for skill in char_dict['skills']:
char.total_sp += skill[2]
cs = CharSkill()
cs.character = char
cs.level = skill[1]
cs.skill_points = skill[2]
base_skill = Skill.objects.filter(name=skill[0]).first()
if not base_skill:
logger.warning("Couldn't find skill %s in db, skipping" % skill[0])
continue
cs.skill = base_skill
cs.typeID = base_skill.typeID
cs.save()
logger.debug("Created CharSkill for {0}".format(skill))
for standing in char_dict['standings']:
corp = NpcCorp.objects.filter(name=standing[0]).first()
if corp:
char_standing = CharStanding.objects.create(character=char,corp=corp, value=standing[1])
char_standing.save()
else:
corp = NpcCorp.objects.create(name=standing[0])
corp.save()
char_standing = CharStanding.objects.create(character=char, corp=corp, value=standing[1])
char_standing.save()
logger.info('Created new npc corp {0}'.format(standing[0]))
char.last_update = now()
char.unspent_skillpoints = char_dict['stats']['unallocated_sp']
char.remaps = char_dict['stats']['remaps']
char.password = char_dict['password'] or ''
char.save()
logger.debug("Character built {0}".format(char_dict['charname']))
return char
def refresh_characters(self, limit, staleness):
staledate = now() - timedelta(days=staleness)
update_limit = now() - timedelta(days=limit)
updated_threads = Thread.objects.filter(last_update__gte=update_limit, blacklisted=False)
stale_characters = Character.objects.filter(
Q(last_update__lte=staledate) | Q(last_update=None), thread__in=updated_threads)
for character in stale_characters:
logger.debug("Updating stale character %s" % character.name)
scraped_info = scrape_character(character.name, character.password)
if scraped_info:
new_sp_total = 0
for skill in scraped_info['skills']:
existing_skill = character.skills.filter(skill__name=skill[0])
if len(existing_skill) > 0:
existing_skill[0].skill_points = skill[2]
existing_skill[0].level = skill[1]
new_sp_total += skill[2]
existing_skill[0].save()
else:
cs = CharSkill()
cs.character = character
if skill[0] in STUPID_OLDNAMELOOKUP:
cs.skill = Skill.objects.filter(name=STUPID_OLDNAMELOOKUP[skill[0]])[0]
else:
cs.skill = Skill.objects.filter(name=skill[0])[0]
cs.level = skill[1]
cs.skill_points = skill[2]
cs.typeID = cs.skill.typeID
cs.save()
character.skills.add(cs)
new_sp_total += skill[2]
character.remaps = scraped_info['stats']['remaps']
character.unspent_skillpoints = scraped_info['stats']['unallocated_sp']
character.total_sp = new_sp_total
character.last_update = now()
character.save()
logger.debug("Update of stale character %s complete" % character.name)
def buildchar(char_dict):
logger.debug("Building new character, %s" % char_dict['charname'])
char = Character()
char.name = char_dict['charname']
char.total_sp = 0
char.save()
for skill in char_dict['skills']:
cs = CharSkill()
cs.character = char
if skill[0] in STUPID_OLDNAMELOOKUP:
cs.skill = Skill.objects.filter(name=STUPID_OLDNAMELOOKUP[skill[0]])[0]
else:
cs.skill = Skill.objects.filter(name=skill[0])[0]
cs.level = skill[1]
cs.skill_points = skill[2]
cs.typeID = cs.skill.typeID
cs.save()
logger.debug("Created CharSkill for {0}".format(skill))
char.skills.add(cs)
char.total_sp += skill[2]
for standing in char_dict['standings']:
corp = NPC_Corp.objects.filter(name=standing[0]).first()
if corp:
char.standings.add(Standing.objects.create(corp=corp, value=standing[1]))
else:
corp = NPC_Corp.objects.create(name=standing[0])
corp.save()
char.standings.add(Standing.objects.create(corp=corp, value=standing[1]))
logger.info('Created new npc corp {0}'.format(standing[0]))
char.last_update = now()
char.unspent_skillpoints = char_dict['stats']['unallocated_sp']
char.remaps = char_dict['stats']['remaps']
char.password = char_dict['password'] or ''
char.save()
logger.debug("Character built {0}".format(char_dict['charname']))
return char
def scrape_eveo(num_pages, update_skills):
threads = []
for x in range(1, num_pages + 1):
threads.extend(get_bazaar_page(x))
for thread in threads:
existing = Thread.objects.filter(thread_id=thread['threadID'])
if len(existing) > 0:
existing[0].last_update = datetime.now()
existing[0].thread_title = thread['title']
if update_skills:
charname, password = scrape_thread(thread)
if charname:
skills = scrape_skills(charname, password)
existing_char = existing[0].character
new_sp_total = 0
for skill in skills:
existing_skill = existing_char.skills.filter(skill__name=skill[0])
if len(existing_skill) > 0:
existing_skill[0].skill_points = skill[2]
existing_skill[0].level = skill[1]
new_sp_total += skill[2]
existing_skill[0].save()
else:
cs = CharSkill()
cs.character = existing_char
if skill[0] in STUPID_OLDNAMELOOKUP:
cs.skill = Skill.objects.filter(name=STUPID_OLDNAMELOOKUP[skill[0]])[0]
else:
cs.skill = Skill.objects.filter(name=skill[0])[0]
cs.level = skill[1]
cs.skill_points = skill[2]
cs.save()
existing_char.skills.add(cs)
new_sp_total = +skill[2]
existing_char.total_sp = new_sp_total
existing_char.save()
existing[0].save()
continue
else:
charname, password = scrape_thread(thread)
if charname:
skills = scrape_skills(charname, password)
standings = scrape_standings(charname, password)
t = Thread()
t.thread_id = thread['threadID']
t.last_update = datetime.now()
t.thread_text = ''
t.thread_title = thread['title']
if skills:
t.blacklisted = False
character = buildchar(charname, skills, standings)
t.character = character
t.save()
else:
t.blacklisted = True
t.save()
if NEW_CORPS:
for char in Character.objects.all():
for corp in NEW_CORPS:
try:
char.standings.get(corp=corp)
except:
char.standings.add(Standing.objects.create(corp=corp, value=0))
char.save()