def buildchar(charname, skills, standings): char = Character() char.name = charname char.total_sp = 0 char.save() for skill in skills: cs = CharSkill() cs.character = char if skill[0] in STUPID_OLDNAMELOOKUP: cs.skill = Skill.objects.filter(name=STUPID_OLDNAMELOOKUP[skill[0]])[0] else: cs.skill = Skill.objects.filter(name=skill[0])[0] cs.level = skill[1] cs.skill_points = skill[2] cs.save() char.skills.add(cs) char.total_sp += skill[2] for standing in standings: try: corp = NPC_Corp.objects.get(name=standing[0]) except ObjectDoesNotExist: corp = NPC_Corp.objects.create(name=standing[0]) print 'Created new npc corp', standing[0] corp.save() global NEW_CORPS NEW_CORPS.append(corp) char.standings.add(Standing.objects.create(corp=corp, value=standing[1])) if standings: for corp in NPC_Corp.objects.all(): try: char.standings.get(corp=corp) except ObjectDoesNotExist: char.standings.add(Standing.objects.create(corp=corp, value=0)) char.save() return char
def refresh_characters(limit, staleness): session = requests.Session() staledate = now() - timedelta(days=staleness) update_limit = now() - timedelta(days=limit) updated_threads = Thread.objects.filter(last_update__gte=update_limit, blacklisted=False) stale_characters = Character.objects.filter( Q(last_update__lte=staledate) | Q(last_update=None), thread__in=updated_threads) for character in stale_characters: logger.debug("Updating stale character %s" % character.name) scraped_info = scrape_character(session, character.name, character.password) if scraped_info: new_sp_total = 0 for skill in scraped_info['skills']: existing_skill = character.skills.filter(skill__name=skill[0]) if len(existing_skill) > 0: existing_skill[0].skill_points = skill[2] existing_skill[0].level = skill[1] new_sp_total += skill[2] existing_skill[0].save() else: cs = CharSkill() cs.character = character cs.level = skill[1] cs.skill_points = skill[2] cs.typeID = cs.skill.typeID cs.save() character.skills.add(cs) new_sp_total += skill[2] character.remaps = scraped_info['stats']['remaps'] character.unspent_skillpoints = scraped_info['stats']['unallocated_sp'] character.total_sp = new_sp_total character.last_update = now() character.save() logger.debug("Update of stale character %s complete" % character.name)
def buildchar(char_dict): logger.debug("Building new character, %s" % char_dict['charname']) char = Character() char.name = char_dict['charname'] char.total_sp = 0 char.save() for skill in char_dict['skills']: cs = CharSkill() cs.character = char if skill[0] in STUPID_OLDNAMELOOKUP: cs.skill = Skill.objects.filter( name=STUPID_OLDNAMELOOKUP[skill[0]])[0] else: cs.skill = Skill.objects.filter(name=skill[0])[0] cs.level = skill[1] cs.skill_points = skill[2] cs.typeID = cs.skill.typeID cs.save() logger.debug("Created CharSkill for {0}".format(skill)) char.skills.add(cs) char.total_sp += skill[2] for standing in char_dict['standings']: corp = NPC_Corp.objects.filter(name=standing[0]).first() if corp: char.standings.add( Standing.objects.create(corp=corp, value=standing[1])) else: corp = NPC_Corp.objects.create(name=standing[0]) corp.save() char.standings.add( Standing.objects.create(corp=corp, value=standing[1])) logger.info('Created new npc corp {0}'.format(standing[0])) char.last_update = now() char.unspent_skillpoints = char_dict['stats']['unallocated_sp'] char.remaps = char_dict['stats']['remaps'] char.password = char_dict['password'] or '' char.save() logger.debug("Character built {0}".format(char_dict['charname'])) return char
def buildchar(char_dict): logger.debug("Building new character, %s" % char_dict['charname']) char = Character() char.name = char_dict['charname'] char.total_sp = 0 char.save() for skill in char_dict['skills']: char.total_sp += skill[2] cs = CharSkill() cs.character = char cs.level = skill[1] cs.skill_points = skill[2] base_skill = Skill.objects.filter(name=skill[0]).first() if not base_skill: logger.warning("Couldn't find skill %s in db, skipping" % skill[0]) continue cs.skill = base_skill cs.typeID = base_skill.typeID cs.save() logger.debug("Created CharSkill for {0}".format(skill)) for standing in char_dict['standings']: corp = NpcCorp.objects.filter(name=standing[0]).first() if corp: char_standing = CharStanding.objects.create(character=char,corp=corp, value=standing[1]) char_standing.save() else: corp = NpcCorp.objects.create(name=standing[0]) corp.save() char_standing = CharStanding.objects.create(character=char, corp=corp, value=standing[1]) char_standing.save() logger.info('Created new npc corp {0}'.format(standing[0])) char.last_update = now() char.unspent_skillpoints = char_dict['stats']['unallocated_sp'] char.remaps = char_dict['stats']['remaps'] char.password = char_dict['password'] or '' char.save() logger.debug("Character built {0}".format(char_dict['charname'])) return char
def refresh_characters(self, limit, staleness): staledate = now() - timedelta(days=staleness) update_limit = now() - timedelta(days=limit) updated_threads = Thread.objects.filter(last_update__gte=update_limit, blacklisted=False) stale_characters = Character.objects.filter( Q(last_update__lte=staledate) | Q(last_update=None), thread__in=updated_threads) for character in stale_characters: logger.debug("Updating stale character %s" % character.name) scraped_info = scrape_character(character.name, character.password) if scraped_info: new_sp_total = 0 for skill in scraped_info['skills']: existing_skill = character.skills.filter(skill__name=skill[0]) if len(existing_skill) > 0: existing_skill[0].skill_points = skill[2] existing_skill[0].level = skill[1] new_sp_total += skill[2] existing_skill[0].save() else: cs = CharSkill() cs.character = character if skill[0] in STUPID_OLDNAMELOOKUP: cs.skill = Skill.objects.filter(name=STUPID_OLDNAMELOOKUP[skill[0]])[0] else: cs.skill = Skill.objects.filter(name=skill[0])[0] cs.level = skill[1] cs.skill_points = skill[2] cs.typeID = cs.skill.typeID cs.save() character.skills.add(cs) new_sp_total += skill[2] character.remaps = scraped_info['stats']['remaps'] character.unspent_skillpoints = scraped_info['stats']['unallocated_sp'] character.total_sp = new_sp_total character.last_update = now() character.save() logger.debug("Update of stale character %s complete" % character.name)
def buildchar(char_dict): logger.debug("Building new character, %s" % char_dict['charname']) char = Character() char.name = char_dict['charname'] char.total_sp = 0 char.save() for skill in char_dict['skills']: cs = CharSkill() cs.character = char if skill[0] in STUPID_OLDNAMELOOKUP: cs.skill = Skill.objects.filter(name=STUPID_OLDNAMELOOKUP[skill[0]])[0] else: cs.skill = Skill.objects.filter(name=skill[0])[0] cs.level = skill[1] cs.skill_points = skill[2] cs.typeID = cs.skill.typeID cs.save() logger.debug("Created CharSkill for {0}".format(skill)) char.skills.add(cs) char.total_sp += skill[2] for standing in char_dict['standings']: corp = NPC_Corp.objects.filter(name=standing[0]).first() if corp: char.standings.add(Standing.objects.create(corp=corp, value=standing[1])) else: corp = NPC_Corp.objects.create(name=standing[0]) corp.save() char.standings.add(Standing.objects.create(corp=corp, value=standing[1])) logger.info('Created new npc corp {0}'.format(standing[0])) char.last_update = now() char.unspent_skillpoints = char_dict['stats']['unallocated_sp'] char.remaps = char_dict['stats']['remaps'] char.password = char_dict['password'] or '' char.save() logger.debug("Character built {0}".format(char_dict['charname'])) return char
def scrape_eveo(num_pages, update_skills): threads = [] for x in range(1, num_pages + 1): threads.extend(get_bazaar_page(x)) for thread in threads: existing = Thread.objects.filter(thread_id=thread['threadID']) if len(existing) > 0: existing[0].last_update = datetime.now() existing[0].thread_title = thread['title'] if update_skills: charname, password = scrape_thread(thread) if charname: skills = scrape_skills(charname, password) existing_char = existing[0].character new_sp_total = 0 for skill in skills: existing_skill = existing_char.skills.filter(skill__name=skill[0]) if len(existing_skill) > 0: existing_skill[0].skill_points = skill[2] existing_skill[0].level = skill[1] new_sp_total += skill[2] existing_skill[0].save() else: cs = CharSkill() cs.character = existing_char if skill[0] in STUPID_OLDNAMELOOKUP: cs.skill = Skill.objects.filter(name=STUPID_OLDNAMELOOKUP[skill[0]])[0] else: cs.skill = Skill.objects.filter(name=skill[0])[0] cs.level = skill[1] cs.skill_points = skill[2] cs.save() existing_char.skills.add(cs) new_sp_total = +skill[2] existing_char.total_sp = new_sp_total existing_char.save() existing[0].save() continue else: charname, password = scrape_thread(thread) if charname: skills = scrape_skills(charname, password) standings = scrape_standings(charname, password) t = Thread() t.thread_id = thread['threadID'] t.last_update = datetime.now() t.thread_text = '' t.thread_title = thread['title'] if skills: t.blacklisted = False character = buildchar(charname, skills, standings) t.character = character t.save() else: t.blacklisted = True t.save() if NEW_CORPS: for char in Character.objects.all(): for corp in NEW_CORPS: try: char.standings.get(corp=corp) except: char.standings.add(Standing.objects.create(corp=corp, value=0)) char.save()