def process_missed_market_transactions(missed_journal_ids, entity_doc, division=None): """ if there are journal entries that are still missing extra context data that comes from 'transactions' endpoint, attempt to update those entries """ if division: entity_doc['missed_market_transactions_' + str(division)] = [] else: entity_doc['missed_market_transactions'] = [] for missed_journal_id in missed_journal_ids: id_filter = {'id': missed_journal_id} result = shared.db.journals.find_one(id_filter) if result is None: logger.error('journal entry in missed market transactions array not found, this should never happen') logger.error('entity with error: ' + str(entity_doc['id']) + ' journal entry error: ' + str(missed_journal_id)) continue if 'context' in result and 'type' in result['context'][0] and result['context'][0]['type'] == 'market_transaction_id': result['context_id_type'] = result['context'][0]['type'] result['context_id'] = result['context'][0]['id'] result['context'] = [{}] context_handler.update_market_transaction(result, entity_doc, division) del result['context_id_type'] del result['context_id'] id_filter = {'id': result['id']} update = {'$set': result} shared.db.journals.update_one(id_filter, update) else: logger.error('journal entry in missed market transactions array has wrong context_id/type, this should never happen.') logger.error('entity with error: ' + str(entity_doc['id']) + ' journal entry error: ' + str(missed_journal_id))
def process_character_wallets(): """ processes character wallets for all characters that have tokens """ try: logger.debug('start character wallet refresh') user_cursor = shared.db.entities.find({ 'tokens': { '$exists': True } }) for user_doc in user_cursor: try: process_character(user_doc) except JournalError: logger.error('Aborting character wallet processing for: ' + str(user_doc['id'])) logger.debug('done character wallet refresh') except Exception as e: logger.exception(e) return
def process_corp(user_doc): """ if the esi token has the correct scope, update the wallet amount and see if it is time for a journal update """ if ('scopes' not in user_doc or 'esi-wallet.read_corporation_wallets.v1' not in user_doc['scopes'].split(' ')): return character_data_to_update = {} if not refresh_token(user_doc, character_data_to_update): return if len(character_data_to_update) > 0: character_filter = {'id': user_doc['id']} update = {"$set": character_data_to_update} shared.db.entities.update_one(character_filter, update) corp_filter = {'id': user_doc['corporation_id']} corp_doc = shared.db.entities.find_one(corp_filter) corp_data_to_update = {} op = shared.esiapp.op['get_corporations_corporation_id_wallets']( corporation_id=corp_doc['id'] ) wallet = shared.esiclient.request(op) if wallet.status != 200: logger.error('status: ' + str(wallet.status) + ' error with getting corp wallet data: ' + str(wallet.data)) logger.error('headers: ' + str(wallet.header)) logger.error('error with getting corp wallet data: ' + str(corp_doc['id'])) if 'error' in wallet.data and wallet.data['error'] == 'Character does not have required role(s)': logger.error('Character ' + user_doc['name'] + ' does not seem to have roles to access corp wallet, removing read_corporation_wallets scope') data_to_update = {} data_to_update['scopes'] = user_doc['scopes'].replace('esi-wallet.read_corporation_wallets.v1', '') data_to_update['scopes'] = data_to_update['scopes'].replace(' ', '').strip() character_filter = {'id': user_doc['id']} update = {"$set": data_to_update} shared.db.entities.update_one(character_filter, update) return corp_data_to_update['wallets'] = wallet.data last_update = datetime.fromtimestamp(corp_doc.get('last_journal_update') or 0.0, timezone.utc) now_utc = datetime.utcnow().replace(tzinfo=timezone.utc) if last_update + timedelta(hours=1) < now_utc: for wallet_division in range(1, 8): missed_journal_string = 'missed_market_transactions_' + str(wallet_division) missed_journal_ids = corp_doc.get(missed_journal_string) or [] process_missed_market_transactions(missed_journal_ids, corp_doc, wallet_division) process_journal(1, corp_doc, wallet_division) corp_data_to_update['last_journal_update'] = now_utc.timestamp() corp_filter = {'id': corp_doc['id']} update = {"$set": corp_data_to_update} shared.db.entities.update_one(corp_filter, update)
def update_all_public_info(): try: logger.debug('start public info refresh') entity_cursor = shared.db.entities.find({}) for entity_doc in entity_cursor: if 'type' not in entity_doc: logger.error('DB entry ' + entity_doc.id + ' does not have a type') continue elif entity_doc['type'] == 'character': shared.user_update(entity_doc['id']) elif entity_doc['type'] == 'corporation': shared.corp_update(entity_doc['id']) elif entity_doc['type'] == 'alliance': shared.alliance_update(entity_doc['id']) logger.debug('done public info refresh') except Exception as e: logger.exception(e) return
def update_constellation(constellation_id): """ adds a previously unknown constellation to the DB and returns it """ data_to_update = {} data_to_update['id'] = constellation_id data_to_update['type'] = 'constellation' op = shared.esiapp.op['get_universe_constellations_constellation_id']( constellation_id=constellation_id) public_data = shared.esiclient.request(op) if public_data.status != 200: logger.error('status: ' + str(public_data.status) + ' error with getting constellation data: ' + str(public_data.data)) logger.error('headers: ' + str(public_data.header)) logger.error('constellation with error: ' + str(constellation_id)) return False, False, False data_to_update['name'] = public_data.data['name'] data_to_update['systems'] = public_data.data['systems'] data_to_update['region_id'] = public_data.data['region_id'] id_filter = {'id': public_data.data['region_id']} result = shared.db.entities.find_one(id_filter) if result is None: region_name = update_region(public_data.data['region_id']) if not region_name: return False, False, False data_to_update['region_name'] = region_name else: data_to_update['region_name'] = result['name'] data_to_update_id = {'id': data_to_update['id']} update = {"$set": data_to_update} shared.db.entities.update_one(data_to_update_id, update, upsert=True) return data_to_update['name'], data_to_update[ 'region_name'], public_data.data['region_id']
def update_station(station_id): """ adds a previously unknown station to the DB and returns it """ if station_id > 100000000: logger.info('Station ID out of range: ' + str(station_id) + ', it is probably a citadel.') return data_to_update = {} data_to_update['id'] = station_id data_to_update['type'] = 'station' op = shared.esiapp.op['get_universe_stations_station_id']( station_id=station_id) public_data = shared.esiclient.request(op) if public_data.status != 200: logger.error('status: ' + str(public_data.status) + ' error with getting station data: ' + str(public_data.data)) logger.error('headers: ' + str(public_data.header)) logger.error('system with error: ' + str(station_id)) return data_to_update['name'] = public_data.data['name'] data_to_update['type_id'] = public_data.data['type_id'] data_to_update['system_id'] = public_data.data['system_id'] id_filter = {'id': public_data.data['system_id']} result = shared.db.entities.find_one(id_filter) if result is None: update_system(public_data.data['system_id']) data_to_update_id = {'id': data_to_update['id']} update = {"$set": data_to_update} return shared.db.entities.find_one_and_update( data_to_update_id, update, upsert=True, return_document=ReturnDocument.AFTER)
def refresh_token(user_doc, data_to_update={}): """ update the ESI token if necessary returns False if the update fails """ access_token_expires = datetime.strptime(user_doc['tokens']['ExpiresOn'], datetime_format) sso_data = { 'access_token': user_doc['tokens']['access_token'], 'refresh_token': user_doc['tokens']['refresh_token'], 'expires_in': ( access_token_expires - datetime.utcnow() ).total_seconds() } shared.esisecurity.update_token(sso_data) if sso_data['expires_in'] <= 30: try: tokens = shared.esisecurity.refresh() except exceptions.SSLError: logger.error('ssl error refreshing token for ' + str(user_doc['id'])) return False except APIException as e: logger.error('error refreshing token for: ' + str(user_doc['id'])) logger.error('error is: ' + str(e)) return False data_to_update['tokens'] = tokens delta_expire = timedelta(seconds=data_to_update['tokens']['expires_in']) token_expire = datetime.utcnow() + delta_expire data_to_update['tokens']['ExpiresOn'] = token_expire.strftime(datetime_format) return True
def process_character(user_doc): """ if the esi token has the correct scope, update the wallet amount and see if it is time for a journal update """ if ('scopes' not in user_doc or 'esi-wallet.read_character_wallet.v1' not in user_doc['scopes'].split(' ')): return data_to_update = {} if not refresh_token(user_doc, data_to_update): return op = shared.esiapp.op['get_characters_character_id_wallet']( character_id=user_doc['id'] ) wallet = shared.esiclient.request(op) if wallet.status != 200: logger.error('status: ' + str(wallet.status) + ' error with getting character wallet data: ' + str(wallet.data)) logger.error('headers: ' + str(wallet.header)) logger.error('error with getting character wallet data: ' + str(user_doc['id'])) return data_to_update['wallet'] = wallet.data last_update = datetime.fromtimestamp(user_doc.get('last_journal_update') or 0.0, timezone.utc) now_utc = datetime.utcnow().replace(tzinfo=timezone.utc) if last_update + timedelta(hours=1) < now_utc: missed_journal_ids = user_doc.get('missed_market_transactions') or [] process_missed_market_transactions(missed_journal_ids, user_doc) process_journal(1, user_doc) data_to_update['last_journal_update'] = now_utc.timestamp() character_filter = {'id': user_doc['id']} update = {"$set": data_to_update} shared.db.entities.update_one(character_filter, update)
def update_system(system_id): """ adds a previously unknown system to the DB and returns it """ data_to_update = {} data_to_update['id'] = system_id data_to_update['type'] = 'system' op = shared.esiapp.op['get_universe_systems_system_id']( system_id=system_id) public_data = shared.esiclient.request(op) if public_data.status != 200: logger.error('status: ' + str(public_data.status) + ' error with getting system data: ' + str(public_data.data)) logger.error('headers: ' + str(public_data.header)) logger.error('system with error: ' + str(system_id)) return data_to_update['name'] = public_data.data['name'] data_to_update['security_status'] = public_data.data['security_status'] data_to_update['constellation_id'] = public_data.data['constellation_id'] if 'stations' in public_data.data: data_to_update['stations'] = public_data.data['stations'] id_filter = {'id': public_data.data['constellation_id']} result = shared.db.entities.find_one(id_filter) if result is None: contellation_name, region_name, region_id = update_constellation( public_data.data['constellation_id']) if not contellation_name: return data_to_update['region_name'] = region_name data_to_update['region_id'] = region_id data_to_update['constellation_name'] = contellation_name else: data_to_update['constellation_name'] = result['name'] data_to_update['region_name'] = result['region_name'] data_to_update['region_id'] = result['region_id'] data_to_update_id = {'id': data_to_update['id']} update = {"$set": data_to_update} return shared.db.entities.find_one_and_update( data_to_update_id, update, upsert=True, return_document=ReturnDocument.AFTER)
def update_region(region_id): """ adds a previously unknown region to the DB and returns it """ data_to_update = {} data_to_update['id'] = region_id data_to_update['type'] = 'region' op = shared.esiapp.op['get_universe_regions_region_id']( region_id=region_id) public_data = shared.esiclient.request(op) if public_data.status != 200: logger.error('status: ' + str(public_data.status) + ' error with getting region data: ' + str(public_data.data)) logger.error('headers: ' + str(public_data.header)) logger.error('region with error: ' + str(region_id)) return False data_to_update['name'] = public_data.data['name'] data_to_update['constellations'] = public_data.data['constellations'] data_to_update['description'] = public_data.data['description'] data_to_update_id = {'id': data_to_update['id']} update = {"$set": data_to_update} shared.db.entities.update_one(data_to_update_id, update, upsert=True) return data_to_update['name']
def update_market_transaction(journal_entry, entity_doc, division): """ adds stations (or locations) and item type to market_transaction_ids """ #CCPls fix bug if journal_entry['context_id'] == 1: return if not division: op = shared.esiapp.op[ 'get_characters_character_id_wallet_transactions']( character_id=entity_doc['id'], from_id=journal_entry['context_id']) else: op = shared.esiapp.op[ 'get_corporations_corporation_id_wallets_division_transactions']( corporation_id=entity_doc['id'], division=division, from_id=journal_entry['context_id']) public_data = shared.esiclient.request(op) if public_data.status != 200: logger.error('status: ' + str(public_data.status) + ' error with getting market transaction: ' + str(public_data.data)) logger.error('headers: ' + str(public_data.header)) logger.error('entity with error: ' + str(entity_doc['id'])) else: for transaction in public_data.data: if transaction['transaction_id'] == journal_entry['context_id']: journal_entry['unit_price'] = transaction['unit_price'] journal_entry['quantity'] = transaction['quantity'] journal_entry['context'][0]['id'] = journal_entry['context_id'] journal_entry['context'][0]['type'] = journal_entry[ 'context_id_type'] journal_entry['context'].append({}) journal_entry['context'][1]['id'] = transaction['location_id'] journal_entry['context'][1]['type'] = 'location_id' journal_entry['context'].append({}) journal_entry['context'][2]['id'] = transaction['type_id'] journal_entry['context'][2]['type'] = 'item' id_filter = {'id': transaction['type_id']} result = shared.db.entities.find_one(id_filter) if result is None: result = update_item(transaction['type_id'], 'item') if result: journal_entry['context'][2]['name'] = result['name'] id_filter = {'id': transaction['location_id']} result = shared.db.entities.find_one(id_filter) if result is None: result = update_station(transaction['location_id']) if result: journal_entry['context'][1]['name'] = result['name'] journal_entry['context'][1]['type'] = result['type'] journal_entry['context'][1]['type_id'] = result['type_id'] if journal_entry[ 'ref_type'] == 'market_escrow' and journal_entry[ 'first_party_id'] == journal_entry[ 'second_party_id']: journal_entry['second_party_id'] = transaction['client_id'] result = shared.decode_party_id( journal_entry['second_party_id']) if result: journal_entry['second_party_name'] = result['name'] journal_entry['second_party_type'] = result['type'] return elif transaction['transaction_id'] < journal_entry['context_id']: break # if ESI fails or the market_transaction simply isn't in the ESI response, # the journal entry is added to a special field so it can be processed on the next journal update logger.info( 'market transaction ID ' + str(journal_entry['context_id']) + ' not found in transaction data, probably bad cache timing. Will try again later.' ) if division: journal_entry['context'][0]['id'] = journal_entry['context_id'] journal_entry['context'][0]['type'] = journal_entry['context_id_type'] missed_journal_ids = entity_doc.get('missed_market_transactions_' + str(division)) or [] missed_journal_ids.append(journal_entry['id']) entity_doc['missed_market_transactions_' + str(division)] = missed_journal_ids else: journal_entry['context'][0]['id'] = journal_entry['context_id'] journal_entry['context'][0]['type'] = journal_entry['context_id_type'] missed_journal_ids = entity_doc.get('missed_market_transactions') or [] missed_journal_ids.append(journal_entry['id']) entity_doc['missed_market_transactions'] = missed_journal_ids
def update_item(item_id, item_type): """ adds a previously unknown item to the DB and returns it """ data_to_update = {} data_to_update['id'] = item_id data_to_update['type'] = item_type op = shared.esiapp.op['get_universe_types_type_id'](type_id=item_id) public_data = shared.esiclient.request(op) if public_data.status != 200: if public_data.status == 400: logger.info('No item found for: ' + str(item_id) + ', it is probably not an item.') return logger.error('status: ' + str(public_data.status) + ' error with getting item data: ' + str(public_data.data)) logger.error('headers: ' + str(public_data.header)) logger.error('item with error: ' + str(item_id)) return data_to_update['name'] = public_data.data['name'] data_to_update['group_id'] = public_data.data['group_id'] id_filter = {'id': public_data.data['group_id']} result = shared.db.entities.find_one(id_filter) if result is None: op = shared.esiapp.op['get_universe_groups_group_id']( group_id=public_data.data['group_id']) public_data = shared.esiclient.request(op) if public_data.status != 200: logger.error('status: ' + str(public_data.status) + ' error with getting group data: ' + str(public_data.data)) logger.error('headers: ' + str(public_data.header)) logger.error('group with error: ' + str(public_data.data['group_id'])) return group_data = {} group_data['id'] = public_data.data['group_id'] group_data['type'] = 'group' group_data['types'] = public_data.data['types'] group_data['name'] = public_data.data['name'] data_to_update_id = {'id': group_data['id']} update = {"$set": group_data} shared.db.entities.update_one(data_to_update_id, update, upsert=True) data_to_update_id = {'id': data_to_update['id']} update = {"$set": data_to_update} return shared.db.entities.find_one_and_update( data_to_update_id, update, upsert=True, return_document=ReturnDocument.AFTER)
def process_journal(page, entity_doc, division=None): """ handles updating the journal entries for both characters and corps, also handles pagination for journals """ if division: last_journal_entry = entity_doc.get('last_journal_entry_' + str(division)) or 0 op = shared.esiapp.op['get_corporations_corporation_id_wallets_division_journal']( corporation_id=entity_doc['id'], division=division, page=page ) else: last_journal_entry = entity_doc.get('last_journal_entry') or 0 op = shared.esiapp.op['get_characters_character_id_wallet_journal']( character_id=entity_doc['id'], page=page ) journal = shared.esiclient.request(op) if journal.status != 200: logger.error('status: ' + str(journal.status) + ' error with getting journal data: ' + str(journal.data)) logger.error('headers: ' + str(journal.header)) logger.error('error with getting journal data: ' + str(entity_doc['id'])) raise JournalError() num_pages = int(journal.header['X-Pages'][0]) if page < num_pages and journal.data[-1]['id'] > last_journal_entry: process_journal(page+1, entity_doc, division) new_journal_entries = [] for journal_entry in journal.data: if journal_entry['id'] > last_journal_entry: if journal_entry['first_party_id'] == entity_doc['id']: journal_entry['first_party_balance'] = journal_entry.pop('balance') journal_entry['first_party_amount'] = journal_entry.pop('amount') journal_entry['second_party_amount'] = journal_entry['first_party_amount'] * -1 if division: journal_entry['first_party_wallet_division'] = division elif journal_entry['second_party_id'] == entity_doc['id']: journal_entry['second_party_balance'] = journal_entry.pop('balance') journal_entry['second_party_amount'] = journal_entry.pop('amount') journal_entry['first_party_amount'] = journal_entry['second_party_amount'] * -1 if division: journal_entry['second_party_wallet_division'] = division elif 'tax_receiver_id' in journal_entry and journal_entry['tax_receiver_id'] == entity_doc['id']: journal_entry['tax_receiver_balance'] = journal_entry.pop('balance') del journal_entry['amount'] if division: journal_entry['tax_receiver_wallet_division'] = division else: logger.info('Journal entry doesnt match any provided entity fields: ' + str(journal_entry)) logger.info('This may be a corp transaction that doesnt have the corp listed') logger.info('Entity with error: ' + str(entity_doc['id'])) if journal_entry['amount'] < 0: journal_entry['second_party_amount'] = abs(journal_entry.pop('amount')) journal_entry['first_party_amount'] = journal_entry['second_party_amount'] * -1 journal_entry['first_party_corp_balance'] = journal_entry.pop('balance') journal_entry['first_party_corp_id'] = entity_doc['id'] journal_entry['first_party_corp_name'] = entity_doc['name'] if division: journal_entry['first_party_corp_wallet_division'] = division else: logger.error(str(journal_entry['id']) + ' is not a corp transaction! This is bad data!') elif journal_entry['amount'] > 0: journal_entry['second_party_amount'] = abs(journal_entry.pop('amount')) journal_entry['first_party_amount'] = journal_entry['second_party_amount'] * -1 journal_entry['second_party_corp_balance'] = journal_entry.pop('balance') journal_entry['second_party_corp_id'] = entity_doc['id'] journal_entry['second_party_corp_name'] = entity_doc['name'] if division: journal_entry['second_party_corp_wallet_division'] = division else: logger.error(str(journal_entry['id']) + ' is not a corp transaction! This is bad data!') else: logger.error('Corp amount is 0! Cant assign the corp to a party, this will lead to bad data.') decode_journal_entry(journal_entry, entity_doc, division) new_journal_entries.append(journal_entry) else: break if len(new_journal_entries) > 0: if division: entity_doc['last_journal_entry' + str(division)] = new_journal_entries[0]['id'] else: entity_doc['last_journal_entry'] = new_journal_entries[0]['id'] for entry in new_journal_entries: id_filter = {'id': entry['id']} update = {'$set': entry} shared.db.journals.update_one(id_filter, update, upsert=True) entity_filter = {'id': entity_doc['id']} update = {"$set": entity_doc} shared.db.entities.update_one(entity_filter, update)