def update(): """ Fetch a /corp/ContactList.xml.aspx api response, parse it and store it to the database. """ LOG.info("fetching /corp/ContactList.xml.aspx...") api_conn = api.connect() corpApi = api_conn.corp.ContactList(characterID=api.get_charID()) api.check_version(corpApi._meta.version) currentTime = timezone.make_aware(corpApi._meta.currentTime, timezone.utc) my_corp = Corporation.objects.mine() # clean existing standings first Standing.objects.filter(corp=my_corp).delete() for contact in corpApi.corporateContactList: Standing.objects.create(corp=my_corp, contactID=contact.contactID, is_corp_contact=True, contactName=contact.contactName, value=contact.standing, ) for contact in corpApi.allianceContactList: Standing.objects.create(corp=my_corp, contactID=contact.contactID, is_corp_contact=False, contactName=contact.contactName, value=contact.standing, ) UpdateDate.mark_updated(model=Standing, date=currentTime) LOG.info("corp standings updated")
def update(): """ Fetch a /corp/ContactList.xml.aspx api response, parse it and store it to the database. """ LOG.info("fetching /corp/ContactList.xml.aspx...") api_conn = api.connect() corpApi = api_conn.corp.ContactList(characterID=api.get_charID()) api.check_version(corpApi._meta.version) currentTime = timezone.make_aware(datetime.utcfromtimestamp( \ corpApi._meta.currentTime), timezone.utc) my_corp = Corporation.objects.mine() # clean existing standings first Standing.objects.filter(corp=my_corp).delete() for contact in corpApi.corporateContactList: Standing.objects.create(corp=my_corp, contactID=contact.contactID, is_corp_contact=True, contactName=contact.contactName, value=contact.standing, ) for contact in corpApi.allianceContactList: Standing.objects.create(corp=my_corp, contactID=contact.contactID, is_corp_contact=False, contactName=contact.contactName, value=contact.standing, ) UpdateDate.mark_updated(model=Standing, date=currentTime) LOG.info("corp standings updated")
def write_results(new_items, old_items, assets_to_locate, currentTime): if len(old_items) > 0: Asset.objects.all().delete() for asset in new_items.values(): asset.save() update_assets_locations(assets_to_locate) update_assets_names() # we store the update time of the table UpdateDate.mark_updated(model=Asset, date=currentTime)
def write_results(new_items, old_items, assets_to_locate, currentTime): if len(old_items) > 0: Asset.objects.all().delete() for asset in new_items.values(): asset.save() update_assets_locations(assets_to_locate) update_assets_names() # we store the update time of the table UpdateDate.mark_updated(model=Asset, date=currentTime)
def update(): """ Updates all wallets with the missing accounting entries since last scan. """ for wallet in Wallet.objects.all(): update_journal_wallet(wallet) for wallet in Wallet.objects.all(): update_transaction_wallet(wallet) UpdateDate.mark_updated(model=JournalEntry, date=timezone.now()) UpdateDate.mark_updated(model=TransactionEntry, date=timezone.now()) LOG.debug("wallets journal updated")
def update(): """ Updates all wallets with the missing accounting entries since last scan. """ # Temporary fix for this new Wallet division (This is the "Mercenary Wallet") for wallet in Wallet.objects.exclude(walletID=10000): update_journal_wallet(wallet) # for wallet in Wallet.objects.all(): update_transaction_wallet(wallet) UpdateDate.mark_updated(model=JournalEntry, date=timezone.now()) UpdateDate.mark_updated(model=TransactionEntry, date=timezone.now()) LOG.debug("wallets journal updated")
def update(): """ Updates all wallets with the missing accounting entries since last scan. """ # Temporary fix for this new Wallet division (This is the "Mercenary Wallet") for wallet in Wallet.objects.exclude(walletID=10000): update_journal_wallet(wallet) # for wallet in Wallet.objects.all(): update_transaction_wallet(wallet) UpdateDate.mark_updated(model=JournalEntry, date=timezone.now()) UpdateDate.mark_updated(model=TransactionEntry, date=timezone.now()) LOG.debug("wallets journal updated")
def update(): """ Update all orders """ LOG.info("fetching /corp/MarketOrders.xml.aspx...") # Connect to EVE API api_conn = api.connect() ordersApi = api_conn.corp.MarketOrders() # checkApiVersion(ordersApi._meta.version) LOG.debug("parsing api response...") processOrders(ordersApi.orders, api_conn) UpdateDate.mark_updated(model=MarketOrder, date=timezone.now())
def update(): """ Update all orders """ LOG.info("fetching /corp/MarketOrders.xml.aspx...") # Connect to EVE API api_conn = api.connect() ordersApi = api_conn.corp.MarketOrders() # checkApiVersion(ordersApi._meta.version) LOG.debug("parsing api response...") processOrders(ordersApi.orders, api_conn) UpdateDate.mark_updated(model=MarketOrder, date=timezone.now())
def update_wallet_divisions(corpApi, currentTime): LOG.debug("WALLET DIVISIONS:") my_corp = Corporation.objects.mine() wallets = CorpWallet.objects.filter(corp=my_corp) for walletDiv in corpApi.walletDivisions: w_id = walletDiv.accountKey w_name = walletDiv.description try: w = wallets.get(wallet_id=w_id) w.name = w_name except CorpWallet.DoesNotExist: w = CorpWallet(corp=my_corp, wallet_id=w_id, name=w_name) LOG.debug(" %s [%s]", w.name, w.wallet_id) w.save() # we store the update time of the table UpdateDate.mark_updated(model=Wallet, date=currentTime)
def update_hangar_divisions(corpApi, currentTime): LOG.debug("HANGAR DIVISIONS:") my_corp = Corporation.objects.mine() hangars = CorpHangar.objects.filter(corp=my_corp) for hangarDiv in corpApi.divisions: h_id = hangarDiv.accountKey h_name = hangarDiv.description try: h = hangars.get(hangar_id=h_id) h.name = h_name except CorpHangar.DoesNotExist: h = CorpHangar(corp=my_corp, hangar_id=h_id, name=h_name) LOG.debug(" %s [%s]", h.name, h.hangar_id) h.save() # we store the update time of the table UpdateDate.mark_updated(model=Hangar, date=currentTime)
def update(): """ Update all contracts """ LOG.info("fetching /corp/Contracts.xml.aspx...") # Connect to EVE API api_conn = api.connect() LOG.debug("Fetching Contracts...") contractsApi = api_conn.corp.Contracts() current_time = timezone.make_aware(contractsApi._meta.currentTime, timezone.utc) cached_until = timezone.make_aware(contractsApi._meta.cachedUntil, timezone.utc) LOG.debug("current time : %s", str(current_time)) LOG.debug("cached util : %s", str(cached_until)) LOG.debug("parsing api response...") process_contracts(contractsApi.contractList, api_conn) UpdateDate.mark_updated(model=Contract, date=timezone.now())
def update(): """ Update all contracts """ LOG.info("fetching /corp/Contracts.xml.aspx...") # Connect to EVE API api_conn = api.connect() LOG.debug("Fetching Contracts...") contractsApi = api_conn.corp.Contracts() current_time = timezone.make_aware(contractsApi._meta.currentTime, timezone.utc) cached_until = timezone.make_aware(contractsApi._meta.cachedUntil, timezone.utc) LOG.debug("current time : %s", str(current_time)) LOG.debug("cached util : %s", str(cached_until)) LOG.debug("parsing api response...") process_contracts(contractsApi.contractList, api_conn) UpdateDate.mark_updated(model=Contract, date=timezone.now())
def storeTitles(oldTitles, newTitles, date): if len(oldTitles) != 0: titleDiffs = getTitleMemberDiffs(oldTitles, newTitles, date) if titleDiffs: for d in titleDiffs: try: d.save() except Database.Warning: # When DEBUG=true, MySQLdb warnings get counted as exceptions, and there's often a "Warning: Field 'id' doesn't have a default value" # thrown here. Ignore it. https://github.com/evecm/ecm/issues/14 pass # we store the update time of the table UpdateDate.mark_updated(model=TitleMemberDiff, date=date) TitleMembership.objects.all().delete() for tm in newTitles.values(): tm.save() # we store the update time of the table UpdateDate.mark_updated(model=TitleMembership, date=date) # if no diff, we do nothing return len(titleDiffs) else: # 1st import, no diff to write for tm in newTitles.values(): tm.save() # we store the update time of the table UpdateDate.mark_updated(model=TitleMembership, date=date) return 0
def storeTitles(oldTitles, newTitles, date): if len(oldTitles) != 0: titleDiffs = getTitleMemberDiffs(oldTitles, newTitles, date) if titleDiffs: for d in titleDiffs: try: d.save() except Database.Warning: # When DEBUG=true, MySQLdb warnings get counted as exceptions, and there's often a "Warning: Field 'id' doesn't have a default value" # thrown here. Ignore it. https://github.com/evecm/ecm/issues/14 pass # we store the update time of the table UpdateDate.mark_updated(model=TitleMemberDiff, date=date) TitleMembership.objects.all().delete() for tm in newTitles.values(): tm.save() # we store the update time of the table UpdateDate.mark_updated(model=TitleMembership, date=date) # if no diff, we do nothing return len(titleDiffs) else: # 1st import, no diff to write for tm in newTitles.values(): tm.save() # we store the update time of the table UpdateDate.mark_updated(model=TitleMembership, date=date) return 0
def update(): """ Retrieve all corp titles, their names and their role composition. If there are changes in the composition of the titles, the changes are also stored in the database. If there's an error, nothing is written in the database """ logger.info("fetching /corp/Titles.xml.aspx...") # connect to eve API api_conn = api.connect() # retrieve /corp/Titles.xml.aspx titlesApi = api_conn.corp.Titles(characterID=api.get_charID()) api.check_version(titlesApi._meta.version) currentTime = timezone.make_aware(titlesApi._meta.currentTime, timezone.utc) cachedUntil = timezone.make_aware(titlesApi._meta.cachedUntil, timezone.utc) logger.debug("current time : %s", str(currentTime)) logger.debug("cached util : %s", str(cachedUntil)) logger.debug("parsing api response...") my_corp = Corporation.objects.mine() newList = [] # we get all the old TitleComposition from the database oldList = list(TitleComposition.objects.all()) for title in titlesApi.titles: newList.extend(parse_one_title(title, my_corp)) diffs = [] if len(oldList) != 0 : diffs = getDiffs(newList, oldList, currentTime) if diffs : for d in diffs: d.save() # we store the update time of the table UpdateDate.mark_updated(model=TitleCompoDiff, date=currentTime) TitleComposition.objects.all().delete() for c in newList: c.save() # we store the update time of the table UpdateDate.mark_updated(model=TitleComposition, date=currentTime) # if no diff, we do nothing else: # 1st import for c in newList: c.save() # we store the update time of the table UpdateDate.mark_updated(model=TitleComposition, date=currentTime) # update titles access levels for t in Title.objects.all(): t.accessLvl = t.get_access_lvl() t.save() logger.info("%d roles in titles parsed, %d changes since last scan", len(newList), len(diffs))
def update(): """ Retrieve all corp titles, their names and their role composition. If there are changes in the composition of the titles, the changes are also stored in the database. If there's an error, nothing is written in the database """ logger.info("fetching /corp/Titles.xml.aspx...") # connect to eve API api_conn = api.connect() # retrieve /corp/Titles.xml.aspx titlesApi = api_conn.corp.Titles(characterID=api.get_charID()) api.check_version(titlesApi._meta.version) currentTime = timezone.make_aware(titlesApi._meta.currentTime, timezone.utc) cachedUntil = timezone.make_aware(titlesApi._meta.cachedUntil, timezone.utc) logger.debug("current time : %s", str(currentTime)) logger.debug("cached util : %s", str(cachedUntil)) logger.debug("parsing api response...") my_corp = Corporation.objects.mine() newList = [] # we get all the old TitleComposition from the database oldList = list(TitleComposition.objects.all()) for title in titlesApi.titles: newList.extend(parse_one_title(title, my_corp)) diffs = [] if len(oldList) != 0 : diffs = getDiffs(newList, oldList, currentTime) if diffs : for d in diffs: d.save() # we store the update time of the table UpdateDate.mark_updated(model=TitleCompoDiff, date=currentTime) TitleComposition.objects.all().delete() for c in newList: c.save() # we store the update time of the table UpdateDate.mark_updated(model=TitleComposition, date=currentTime) # if no diff, we do nothing else: # 1st import for c in newList: c.save() # we store the update time of the table UpdateDate.mark_updated(model=TitleComposition, date=currentTime) # update titles access levels for t in Title.objects.all(): t.accessLvl = t.get_access_lvl() t.save() logger.info("%d roles in titles parsed, %d changes since last scan", len(newList), len(diffs))
def storeRoles(oldRoles, newRoles, date): if len(oldRoles) != 0: roleDiffs = getRoleMemberDiffs(oldRoles, newRoles, date) if roleDiffs: for d in roleDiffs: d.save() # we store the update time of the table UpdateDate.mark_updated(model=RoleMemberDiff, date=date) RoleMembership.objects.all().delete() for rm in newRoles.values(): rm.save() # we store the update time of the table UpdateDate.mark_updated(model=RoleMembership, date=date) # if no diff, we do nothing return len(roleDiffs) else: # 1st import, no diff to write for rm in newRoles.values(): rm.save() # we store the update time of the table UpdateDate.mark_updated(model=RoleMembership, date=date) return 0
def storeRoles(oldRoles, newRoles, date): if len(oldRoles) != 0: roleDiffs = getRoleMemberDiffs(oldRoles, newRoles, date) if roleDiffs: for d in roleDiffs: d.save() # we store the update time of the table UpdateDate.mark_updated(model=RoleMemberDiff, date=date) RoleMembership.objects.all().delete() for rm in newRoles.values(): rm.save() # we store the update time of the table UpdateDate.mark_updated(model=RoleMembership, date=date) # if no diff, we do nothing return len(roleDiffs) else: # 1st import, no diff to write for rm in newRoles.values(): rm.save() # we store the update time of the table UpdateDate.mark_updated(model=RoleMembership, date=date) return 0
def storeTitles(oldTitles, newTitles, date): if len(oldTitles) != 0: titleDiffs = getTitleMemberDiffs(oldTitles, newTitles, date) if titleDiffs: for d in titleDiffs: d.save() # we store the update time of the table UpdateDate.mark_updated(model=TitleMemberDiff, date=date) TitleMembership.objects.all().delete() for tm in newTitles.values(): tm.save() # we store the update time of the table UpdateDate.mark_updated(model=TitleMembership, date=date) # if no diff, we do nothing return len(titleDiffs) else: # 1st import, no diff to write for tm in newTitles.values(): tm.save() # we store the update time of the table UpdateDate.mark_updated(model=TitleMembership, date=date) return 0
def storeTitles(oldTitles, newTitles, date): if len(oldTitles) != 0: titleDiffs = getTitleMemberDiffs(oldTitles, newTitles, date) if titleDiffs: for d in titleDiffs: d.save() # we store the update time of the table UpdateDate.mark_updated(model=TitleMemberDiff, date=date) TitleMembership.objects.all().delete() for tm in newTitles.values(): tm.save() # we store the update time of the table UpdateDate.mark_updated(model=TitleMembership, date=date) # if no diff, we do nothing return len(titleDiffs) else: # 1st import, no diff to write for tm in newTitles.values(): tm.save() # we store the update time of the table UpdateDate.mark_updated(model=TitleMembership, date=date) return 0
def update_corp_info(corpApi, currentTime): try: try: try: alliance = Alliance.objects.get(allianceID = corpApi.allianceID) except Alliance.DoesNotExist: LOG.info("Adding new Alliance: "+ corpApi.allianceName) alliance = Alliance() alliance.allianceID = corpApi.allianceID alliance.name = corpApi.allianceName alliancesApi = api.connect().eve.AllianceList() for a in alliancesApi.alliances: if a.allianceID == corpApi.allianceID: alliance.shortName = a.shortName alliance.save() break except api.Error: LOG.exception("Failed to fetch AllianceList.xml.aspx from EVE API server") corp = Corporation.objects.mine() alliance = None except: alliance = None description = fix_description(corpApi.description) # reset all other corps Corporation.objects.exclude(corporationID=corpApi.corporationID).update(is_my_corp=False) try: # try to retrieve the db stored corp info corp = Corporation.objects.get(corporationID=corpApi.corporationID) corp.is_my_corp = True corp.corporationID = corpApi.corporationID corp.corporationName = corpApi.corporationName corp.ticker = corpApi.ticker corp.ceoID = corpApi.ceoID corp.ceoName = corpApi.ceoName corp.stationID = corpApi.stationID corp.stationName = corpApi.stationName corp.alliance = alliance corp.description = description corp.taxRate = corpApi.taxRate corp.memberLimit = corpApi.memberLimit except Corporation.DoesNotExist: LOG.debug('First scan, creating corp...') # no corp parsed yet corp = Corporation(is_my_corp = True, corporationID = corpApi.corporationID, corporationName = corpApi.corporationName, ticker = corpApi.ticker, ceoID = corpApi.ceoID, ceoName = corpApi.ceoName, stationID = corpApi.stationID, stationName = corpApi.stationName, description = description, alliance = alliance, taxRate = corpApi.taxRate, memberLimit = corpApi.memberLimit ) if settings.USE_HTTPS: corp.ecm_url = 'https://' + settings.EXTERNAL_HOST_NAME else: corp.ecm_url = 'http://' + settings.EXTERNAL_HOST_NAME if not (corp.private_key and corp.public_key and corp.key_fingerprint): # as this is the first time, we must generate the RSA keypair of our own corp LOG.debug('Generating RSA key pair...') corp.private_key = crypto.generate_rsa_keypair() corp.public_key = crypto.extract_public_key(corp.private_key) corp.key_fingerprint = crypto.key_fingerprint(corp.public_key) LOG.info('Generated RSA key pair for corporation ID %d.' % corpApi.corporationID) corp.save() # we store the update time of the table UpdateDate.mark_updated(model=Corporation, date=currentTime) return corp
def update(): """ Retrieve all corp members, with all basic information about them. If some members have left or have arrived we also store the diff in the database. If there's an error, nothing is written in the database """ LOG.info("fetching /corp/MemberTracking.xml.aspx...") # connect to eve API api_conn = api.connect() # retrieve /corp/MemberTracking.xml.aspx membersApi = api_conn.corp.MemberTracking(characterID=api.get_charID(), extended=1) api.check_version(membersApi._meta.version) currentTime = timezone.make_aware(membersApi._meta.currentTime, timezone.utc) cachedUntil = timezone.make_aware(membersApi._meta.cachedUntil, timezone.utc) LOG.debug("current time : %s", str(currentTime)) LOG.debug("cached util : %s", str(cachedUntil)) LOG.debug("parsing api response...") newMembers = {} oldMembers = {} notCorped = {} oldAccessLvls = {} oldOwners = {} my_corp = Corporation.objects.mine() # we get the old member list from the database for m in Member.objects.all(): if m.corp == my_corp: oldMembers[m] = m else: notCorped[m] = m oldAccessLvls[m.characterID] = m.accessLvl oldOwners[m.characterID] = m.owner for member in membersApi.members: m = parseOneMember(member, my_corp) session = MemberSession(character_id=m.characterID, session_begin=m.lastLogin, session_end=m.lastLogoff, session_seconds=(member.logoffDateTime - member.logonDateTime).seconds) dbsession = MemberSession.objects.filter(character_id=m.characterID, session_begin=m.lastLogin) if len(dbsession) == 0: session.save() newMembers[m] = m diffs, leaved = getDiffs(oldMembers, newMembers, currentTime) # "leaved" is the list of members that leaved (not a list of MemberDiff but real Character objects) # If we delete the old members each time, then all the diffs in roles/titles will not match # as the foreign keys will be gone from the members table... for L in leaved: L.corp = None newMembers[L] = L LOG.info("%d members parsed, %d changes since last scan", len(newMembers), len(diffs)) for m in notCorped.values(): try: # if the previously "not corped" members can now be found in the "new members" # we do nothing newMembers[m] except KeyError: # if the previously "not corped" members still cannot be found in the "new members" # we add them again to the members list newMembers[m] = m for m in newMembers.values(): try: # we restore the old access levels from the database m.accessLvl = oldAccessLvls[m.characterID] m.owner = oldOwners[m.characterID] except KeyError: # 'm' is a brand new member, his/her access level didn't exist before # we leave it to the default value '0' continue for m in newMembers.values(): # to be sure to store the nicknames change, etc. # even if there are no diff, we always overwrite the members m.save() if len(oldMembers) > 0 and len(diffs) > 0: for d in diffs: d.save() # we store the update time of the table UpdateDate.mark_updated(model=MemberDiff, date=currentTime) # we store the update time of the table UpdateDate.mark_updated(model=Member, date=currentTime)
def write_diff_results(diffs, currentTime): for assetDiff in diffs: assetDiff.save() # we store the update time of the table UpdateDate.mark_updated(model=AssetDiff, date=currentTime) LOG.info("%d changes since last scan", len(diffs))
def update(): """ Retrieve all corp members, with all basic information about them. If some members have left or have arrived we also store the diff in the database. If there's an error, nothing is written in the database """ LOG.info("fetching /corp/MemberTracking.xml.aspx...") # connect to eve API api_conn = api.connect() # retrieve /corp/MemberTracking.xml.aspx membersApi = api_conn.corp.MemberTracking(characterID=api.get_charID(), extended=1) api.check_version(membersApi._meta.version) currentTime = timezone.make_aware(membersApi._meta.currentTime, timezone.utc) cachedUntil = timezone.make_aware(membersApi._meta.cachedUntil, timezone.utc) LOG.debug("current time : %s", str(currentTime)) LOG.debug("cached util : %s", str(cachedUntil)) LOG.debug("parsing api response...") newMembers = {} oldMembers = {} notCorped = {} oldAccessLvls = {} oldOwners = {} my_corp = Corporation.objects.mine() # we get the old member list from the database for m in Member.objects.all(): if m.corp == my_corp: oldMembers[m] = m else: notCorped[m] = m oldAccessLvls[m.characterID] = m.accessLvl oldOwners[m.characterID] = m.owner for member in membersApi.members: m = parseOneMember(member, my_corp) session = MemberSession(character_id = m.characterID, session_begin = m.lastLogin, session_end = m.lastLogoff, session_seconds = (member.logoffDateTime-member.logonDateTime).seconds) dbsession = MemberSession.objects.filter(character_id = m.characterID, session_begin = m.lastLogin) if len(dbsession) == 0: session.save() newMembers[m] = m diffs, leaved = getDiffs(oldMembers, newMembers, currentTime) # "leaved" is the list of members that leaved (not a list of MemberDiff but real Character objects) # If we delete the old members each time, then all the diffs in roles/titles will not match # as the foreign keys will be gone from the members table... for L in leaved: L.corp = None newMembers[L] = L LOG.info("%d members parsed, %d changes since last scan", len(newMembers), len(diffs)) for m in notCorped.values(): try: # if the previously "not corped" members can now be found in the "new members" # we do nothing newMembers[m] except KeyError: # if the previously "not corped" members still cannot be found in the "new members" # we add them again to the members list newMembers[m] = m for m in newMembers.values(): try: # we restore the old access levels from the database m.accessLvl = oldAccessLvls[m.characterID] m.owner = oldOwners[m.characterID] except KeyError: # 'm' is a brand new member, his/her access level didn't exist before # we leave it to the default value '0' continue for m in newMembers.values(): # to be sure to store the nicknames change, etc. # even if there are no diff, we always overwrite the members m.save() if len(oldMembers) > 0 and len(diffs) > 0 : for d in diffs: d.save() # we store the update time of the table UpdateDate.mark_updated(model=MemberDiff, date=currentTime) # we store the update time of the table UpdateDate.mark_updated(model=Member, date=currentTime)
def write_diff_results(diffs, currentTime): for assetDiff in diffs: assetDiff.save() # we store the update time of the table UpdateDate.mark_updated(model=AssetDiff, date=currentTime) LOG.info("%d changes since last scan", len(diffs))