def skip_scope(token_scope): """ Return True if we must skip that token_scope This function return True in the following cases: - The user didn't log in for more than 30days - The user didn't log in for more than 7days, and we already updated the token once today - the scope is not registered in the task scopes list - the scope has no tasks registered to it """ yesterday = (utcnow() - datetime.timedelta(days=1)) past_week = (utcnow() - datetime.timedelta(days=7)) past_month = (utcnow() - datetime.timedelta(days=30)) # if the user was not logged in the previous 7 days, update once/day # and if not logged from more than 30 days, don't update if token_scope.user.main_character is None: last_seen = token_scope.user.current_login_at else: last_seen = token_scope.user.main_character.current_login_at if last_seen < past_week: if last_seen < past_month: return True if not token_scope.last_update: return True if token_scope.last_update > yesterday: return True # if no task is defined for the scope, skip it if (token_scope.scope not in CHAR_TASK_SCOPE or CHAR_TASK_SCOPE[token_scope.scope] is None): return True # if nothing match, return False return False
def task_purge(self): """ Purge all old stuff everywhere. """ self.start() # purge all old TaskState objects purge_date_limit = utcnow() - datetime.timedelta( days=config.PURGE_OLD_TASKS) TaskState.query.filter(TaskState.end_date < purge_date_limit).delete() db.session.commit() # purge all tokens people never updated purge_date_limit = utcnow() - datetime.timedelta( days=config.PURGE_INVALID_TOKENS) TokenScope.query.filter_by(valid=False).filter( ((TokenScope.last_update.is_(None)) & (TokenScope.updated_at <= purge_date_limit)) | (TokenScope.last_update < purge_date_limit)).delete() db.session.commit() # purge old market data purge_date_limit = utcnow() - datetime.timedelta( days=config.PURGE_OLD_PRICES) ItemPrice.query.filter(ItemPrice.updated_at < purge_date_limit).delete() db.session.commit() self.end(TaskState.SUCCESS)
def task_purge(self): """ Purge all old stuff everywhere. """ self.start() # purge all old TaskState objects purge_date_limit = utcnow() - datetime.timedelta(days=config.PURGE_OLD_TASKS) TaskState.query.filter(TaskState.end_date < purge_date_limit).delete() db.session.commit() # purge all tokens people never updated purge_date_limit = utcnow() - datetime.timedelta(days=config.PURGE_INVALID_TOKENS) TokenScope.query.filter_by( valid=False ).filter( ((TokenScope.last_update.is_(None)) & (TokenScope.updated_at <= purge_date_limit)) | (TokenScope.last_update < purge_date_limit) ).delete() db.session.commit() # purge old market data purge_date_limit = utcnow() - datetime.timedelta(days=config.PURGE_OLD_PRICES) ItemPrice.query.filter(ItemPrice.updated_at < purge_date_limit).delete() db.session.commit() self.end(TaskState.SUCCESS)
def update_token_state(token, expires_header): """ update the token """ token.request_try = 0 token.last_update = utcnow() token.cached_until = datetime(*parsedate(expires_header)[:6]).replace( tzinfo=pytz.utc) db.session.commit()
def spawn_market_price_tasks(self): """Celery task to spawn market prices update tasks""" self.start() region_list = Region.query.filter(Region.id.in_( config.ESI_REGION_PRICE)).all() for region in region_list: if not is_task_running(region.id, task_update_region_order_price.__name__): item_id_list = [ it[0] for it in db.session.query(ItemPrice.item_id).filter_by( region_id=region.id) ] task_id = "%s-%s-%s" % (utcnow().strftime('%Y%m%d-%H%M%S'), task_update_region_order_price.__name__, region.name) token_state = TaskState( task_id=task_id, id=region.id, scope=task_update_region_order_price.__name__, ) db.session.add(token_state) db.session.commit() task_update_region_order_price.s( region.id, item_id_list).apply_async(task_id=task_id) self.end(TaskState.SUCCESS)
def spawn_character_tasks(): """ Task triggered every minutes that scan all tasks done to find any character based task to do (based on the cached_until field) """ now = utcnow() # checking if API is up. If not, just stop it if not is_server_online(): logger.info('Looks like EVE is still down / in VIP mode. Skipping !') return all_tokens = TokenScope.query.filter_by(valid=True).all() for token_scope in all_tokens: if skip_scope(token_scope): continue # check if there is no running task, and the data is not still cached if (not is_task_running(token_scope.user_id, token_scope.scope) and (not token_scope.cached_until or token_scope.cached_until <= now)): task = CHAR_TASK_SCOPE[token_scope.scope] task_id = "%s-%s-%s" % (now.strftime('%Y%m%d-%H%M%S'), task.__name__, token_scope.user_id) token_state = TaskState( task_id=task_id, id=token_scope.user_id, scope=token_scope.scope, ) db.session.add(token_state) db.session.commit() task.s(token_scope.user_id).apply_async(task_id=task_id)
def get_sso_data(self): return { 'access_token': self.access_token, 'refresh_token': self.refresh_token, 'expires_in': (self.access_token_expires - utcnow()).total_seconds() }
def spawn_universe_tasks(): """ Task triggered every XX minutes (not less than 5) that trigger 'universe' tasks (market prices, industry indexes, ...) """ now = utcnow() # checking if API is up. If not, just stop it if not is_server_online(): logger.info('Looks like EVE is still down / in VIP mode. Skipping !') return for task in UNIVERSE_TASKS: if not is_task_running(None, task.__name__): task_id = "%s-%s" % ( now.strftime('%Y%m%d-%H%M%S'), task.__name__, ) token_state = TaskState( task_id=task_id, id=None, scope=task.__name__, ) db.session.add(token_state) db.session.commit() task.s().apply_async(task_id=task_id)
def spawn_character_tasks(): """ Task triggered every minutes that scan all tasks done to find any character based task to do (based on the cached_until field) """ now = utcnow() all_tokens = TokenScope.query.all() for token_scope in all_tokens: if skip_scope(token_scope): continue # check if there is no running task, and the data is not still cached if ((not token_scope.cached_until or token_scope.cached_until <= now) and not is_task_running(token_scope.user_id, token_scope.scope)): task = CHAR_TASK_SCOPE[token_scope.scope] task_id = "%s-%s-%s" % (now.strftime('%Y%m%d-%H%M%S'), task.__name__, token_scope.user_id) token_state = TaskState( task_id=task_id, id=token_scope.user_id, scope=token_scope.scope, ) db.session.add(token_state) db.session.commit() task.s(token_scope.user_id).apply_async(task_id=task_id)
def get_sso_data(self): return { 'access_token': self.access_token, 'refresh_token': self.refresh_token, 'expires_in': ( self.access_token_expires - utcnow() ).total_seconds() }
def start(self): task_state = TaskState.query.get(self.request.id) if task_state: task_state.start_date = utcnow() task_state.state = TaskState.RUNNING try: db.session.commit() except: db.session.rollback()
def start(self): task_state = TaskState.query.get(self.request.id) if task_state: task_state.start_date = utcnow() task_state.state = TaskState.RUNNING try: db.session.commit() except: logger.exception('[start] Something went wrong while ' 'updating task state') db.session.rollback()
def end(self, state): task_state = TaskState.query.get(self.request.id) if task_state: task_state.end_date = utcnow() if state in TaskState.STATES: task_state.state = state else: task_state.state = TaskState.UNKNOWN try: db.session.commit() except: db.session.rollback()
def task_update_character_skills(self, character_id): """ Update the skills for a given character_id """ self.start() skill_number = 0 character = User.query.get(character_id) if character is None: return # get token token = self.get_token_update_esipy( character_id=character_id, scope=TokenScope.SCOPE_SKILL ) # get current character skills from ESI character_skills = esiclient.request( get_characters_skills(character_id=character_id), ) if character_skills.status == 200: for skill_object in character_skills.data.skills: char_skill = character.skills.filter( Skill.skill_id == skill_object.skill_id ).one_or_none() if char_skill: char_skill.level = skill_object.active_skill_level else: skill = Skill( character=character, skill_id=skill_object.skill_id, level=skill_object.active_skill_level, ) db.session.merge(skill) skill_number += 1 db.session.commit() else: self.inc_fail_token_scope(token, character_skills.status) self.end(TaskState.ERROR) return # update the token and the state token.request_try = 0 token.last_update = utcnow() token.cached_until = datetime( *parsedate(character_skills.header['Expires'][0])[:6] ).replace(tzinfo=pytz.utc) db.session.commit() self.end(TaskState.SUCCESS)
def task_purge(): """ Purge all old stuff everywhere. """ # purge all tokens people never updated try: purge_date_limit = ( utcnow() - datetime.timedelta(days=config.PURGE_INVALID_TOKENS)) TokenScope.query.filter_by(valid=False).filter( ((TokenScope.last_update.is_(None)) & (TokenScope.updated_at <= purge_date_limit)) | (TokenScope.last_update < purge_date_limit)).delete() db.session.commit() # purge old market data purge_date_limit = (utcnow() - datetime.timedelta(days=config.PURGE_OLD_PRICES)) ItemPrice.query.filter( ItemPrice.updated_at < purge_date_limit).delete() db.session.commit() except SQLAlchemyError: db.session.rollback() logger.exception("Error while trying to purge data")
def end(self, state): task_state = TaskState.query.get(self.request.id) if task_state: task_state.end_date = utcnow() if state in TaskState.STATES: task_state.state = state else: task_state.state = TaskState.UNKNOWN try: db.session.commit() except: logger.exception('[end] Something went wrong while ' 'updating task state') db.session.rollback()
def task_update_character_skills(self, character_id): """ Update the skills for a given character_id """ self.start() skill_number = 0 character = User.query.get(character_id) if character is None: return # get token token = self.get_token_update_esipy(character_id=character_id, scope=TokenScope.SCOPE_SKILL) # get current character skills from ESI character_skills = esiclient.request( get_characters_skills(character_id=character_id), ) if character_skills.status == 200: for skill_object in character_skills.data.skills: char_skill = character.skills.filter( Skill.skill_id == skill_object.skill_id).one_or_none() if char_skill: char_skill.level = skill_object.active_skill_level else: skill = Skill( character=character, skill_id=skill_object.skill_id, level=skill_object.active_skill_level, ) db.session.merge(skill) skill_number += 1 db.session.commit() else: self.inc_fail_token_scope(token, character_skills.status) self.end(TaskState.ERROR) return # update the token and the state token.request_try = 0 token.last_update = utcnow() token.cached_until = datetime( *parsedate(character_skills.header['Expires'][0])[:6]).replace( tzinfo=pytz.utc) db.session.commit() self.end(TaskState.SUCCESS)
def spawn_universe_tasks(): """ Task triggered every XX minutes (not less than 5) that trigger 'universe' tasks (market prices, industry indexes, ...) """ now = utcnow() for task in UNIVERSE_TASKS: if not is_task_running(None, task.__name__): task_id = "%s-%s" % ( now.strftime('%Y%m%d-%H%M%S'), task.__name__, ) token_state = TaskState( task_id=task_id, id=None, scope=task.__name__, ) db.session.add(token_state) db.session.commit() task.s().apply_async(task_id=task_id)
def run(self, character, universe, purge): if character: spawn_character_tasks() if universe: spawn_universe_tasks() if purge: task_id = "%s-%s" % ( utcnow().strftime('%Y%m%d-%H%M%S'), task_purge.__name__, ) token_state = TaskState( task_id=task_id, id=None, scope=task_purge.__name__, ) db.session.add(token_state) db.session.commit() task_purge.s().apply_async(task_id=task_id)
def update_itemlist_from_order(region_id, item_list, item_id_list, order): item_id = order['type_id'] # values if we already have this item in database or not # we need custom field label for update, as we don't want the # region_id item_id to be updated but we need them in where clause if item_id in item_id_list: stmt_type = 'update' region_id_label = 'u_region_id' item_id_label = 'u_item_id' else: stmt_type = 'insert' region_id_label = 'region_id' item_id_label = 'item_id' # do we already looped on this item ? if item_id not in item_list[stmt_type]: item_list[stmt_type][item_id] = { 'sell_price': None, 'buy_price': 0, region_id_label: region_id, item_id_label: item_id, 'updated_at': utcnow() } current_item = item_list[stmt_type][item_id] if not order['is_buy_order']: if current_item['sell_price'] is None: current_item['sell_price'] = order['price'] current_item['sell_price'] = min( current_item['sell_price'], order['price'] ) else: current_item['buy_price'] = max( current_item['buy_price'], order['price'] )
def check_and_update_user(): """ check for invalid token and print message and update last seen """ if flask_login.current_user.is_authenticated and not request.is_xhr: char_id = flask_login.current_user.character_id current_user = flask_login.current_user count_error = TokenScope.query.filter_by( valid=False).join(User).filter(( (User.main_character_id.is_(None)) & (User.character_id == char_id) ) | (User.main_character_id == char_id)).filter( ((TokenScope.last_update.is_(None)) & (TokenScope.updated_at >= current_user.current_login_at)) | (TokenScope.last_update >= current_user.current_login_at) ).count() if count_error > 0: flash( 'You have at least one scope that have been invalidate.' ' Please take a moment to check and update it, ' ' or remove it.', 'danger') flask_login.current_user.current_login_at = utcnow() db.session.commit()
def spawn_market_price_tasks(self): """Celery task to spawn market prices update tasks""" self.start() region_list = Region.query.filter( Region.id.in_(config.ESI_REGION_PRICE) ).all() for region in region_list: if not is_task_running(region.id, task_update_region_order_price.__name__): item_id_list = [ it[0] for it in db.session.query( ItemPrice.item_id ).filter_by(region_id=region.id) ] task_id = "%s-%s-%s" % ( utcnow().strftime('%Y%m%d-%H%M%S'), task_update_region_order_price.__name__, region.name ) token_state = TaskState( task_id=task_id, id=region.id, scope=task_update_region_order_price.__name__, ) db.session.add(token_state) db.session.commit() task_update_region_order_price.s( region.id, item_id_list ).apply_async( task_id=task_id ) self.end(TaskState.SUCCESS)
def check_and_update_user(): """ check for invalid token and print message and update last seen """ if flask_login.current_user.is_authenticated and not request.is_xhr: char_id = flask_login.current_user.character_id current_user = flask_login.current_user count_error = TokenScope.query.filter_by( valid=False ).join(User).filter( ((User.main_character_id.is_(None)) & (User.character_id == char_id)) | (User.main_character_id == char_id) ).filter( ((TokenScope.last_update.is_(None)) & (TokenScope.updated_at >= current_user.current_login_at)) | (TokenScope.last_update >= current_user.current_login_at) ).count() if count_error > 0: flash('You have at least one scope that have been invalidate.' ' Please take a moment to check and update it, ' ' or remove it.', 'danger') flask_login.current_user.current_login_at = utcnow() db.session.commit()
def spawn_character_tasks(): """ Task triggered every minutes that scan all tasks done to find any character based task to do (based on the cached_until field) """ now = utcnow() # checking if API is up. If not, just stop it if not is_server_online(): logger.info('Looks like EVE is still down / in VIP mode. Skipping !') return all_tokens = TokenScope.query.filter_by(valid=True).all() for token_scope in all_tokens: if skip_scope(token_scope): continue # check if there is no running task, and the data is not still cached if (not is_task_running(token_scope.user_id, token_scope.scope) and (not token_scope.cached_until or token_scope.cached_until <= now)): task = CHAR_TASK_SCOPE[token_scope.scope] task_id = "%s-%s-%s" % ( now.strftime('%Y%m%d-%H%M%S'), task.__name__, token_scope.user_id ) token_state = TaskState( task_id=task_id, id=token_scope.user_id, scope=token_scope.scope, ) db.session.add(token_state) db.session.commit() task.s(token_scope.user_id).apply_async(task_id=task_id)
def get_delta_update(self): return self.updated_at - utcnow()
def task_update_character_blueprints(self, character_id): """ Update the skills for a given character_id """ self.start() character = User.query.get(character_id) if character is None: return # get token token = self.get_token_update_esipy(character_id=character_id, scope=TokenScope.SCOPE_CHAR_ASSETS) # get current blueprints bps = Blueprint.query.filter_by(character_id=character_id).filter_by( corporation=False).all() blueprints = {} for bp in bps: key = "%s-%d-%d-%d" % (bp.item_id, bp.original, bp.material_efficiency, bp.time_efficiency) # update run to 0, to have the real total run for bpc if not bp.original: bp.total_runs = 0 blueprints[key] = bp # set of known blueprints blueprint_init_list = set(blueprints.keys()) blueprint_updated_list = set() try: # init evelink api = evelink.api.API(sso_token=(token.access_token, 'character')) char = evelink.char.Char(char_id=character.character_id, api=api) api_bp_list = char.blueprints() for blueprint in api_bp_list.result.values(): original = blueprint['quantity'] != -2 runs = blueprint['runs'] me = blueprint['material_efficiency'] te = blueprint['time_efficiency'] item_id = blueprint['type_id'] key = "%s-%d-%d-%d" % (item_id, original, me, te) if key not in blueprint_updated_list: blueprint_updated_list.add(key) if key not in blueprints: blueprints[key] = Blueprint( item_id=item_id, original=original, total_runs=runs, material_efficiency=me, time_efficiency=te, character_id=character_id, ) db.session.add(blueprints[key]) continue if not original: blueprints[key].total_runs += runs # delete every blueprint that have not been updated for key in (blueprint_init_list - blueprint_updated_list): db.session.delete(blueprints[key]) # update the token and the state token.request_try = 0 token.last_update = utcnow() token.cached_until = datetime.fromtimestamp(api_bp_list.expires, tz=pytz.utc) db.session.commit() self.end(TaskState.SUCCESS) except evelink.api.APIError as e: self.inc_fail_token_scope(token, e.code, True) logger.exception(e.message) self.end(TaskState.ERROR) except requests.HTTPError as e: self.inc_fail_token_scope(token, e.response.status_code) logger.exception(e.message) self.end(TaskState.ERROR)
def task_update_region_order_price(self, region_id, item_id_list): """ Get the price from the API and update the database for a given region """ self.start() # call the market page and extract all items from every pages if required page = 0 fails = 0 item_list = {'update': {}, 'insert': {}} expire = utcnow() while True: page += 1 # try up to 3 times to get the data, else go to next page for retry in xrange(3): op = get_markets_region_id_orders(region_id=region_id, order_type='all', page=page) region_orders_res = esiclient.request(op, raw_body_only=True) logger.debug('Request #%d %s [%d]' % (retry, op[0].url, region_orders_res.status)) if region_orders_res.status == 200: break if region_orders_res.status != 200: fails += 1 logger.error('Request failed after 3 tries [%s, %s, %d]: %s' % ( op[0].url, op[0].query, region_orders_res.status, region_orders_res.raw, )) # if we have more than 2 fails, we stop gathering data as there # are too many missing page (when average page is around 2-3) if fails > 2: break else: continue region_orders = json.loads(region_orders_res.raw) if not region_orders: break for order in region_orders: update_itemlist_from_order(region_id, item_list, item_id_list, order) try: save_item_prices(item_list) except SQLAlchemyError as e: logger.error( 'Something went wrong while trying to insert/update data: %s' % (e.message)) db.session.rollback() fails += 1 if fails > 0: self.end(TaskState.ERROR) else: self.end(TaskState.SUCCESS)
def task_update_character_blueprints(self, character_id): """ Update the skills for a given character_id """ self.start() character = User.query.get(character_id) if character is None: return # get token token = self.get_token_update_esipy(character_id=character_id, scope=TokenScope.SCOPE_CHAR_BLUEPRINTS) # get current blueprints bps = Blueprint.query.filter_by(character_id=character_id).filter_by( corporation=False).all() blueprints = {} for bp in bps: key = "%s-%d-%d-%d" % (bp.item_id, bp.original, bp.material_efficiency, bp.time_efficiency) # update run to 0, to have the real total run for bpc if not bp.original: bp.total_runs = 0 blueprints[key] = bp # set of known blueprints blueprint_init_list = set(blueprints.keys()) blueprint_updated_list = set() # get the first page to have the page number op_blueprint = get_characters_blueprints(character_id=character_id, page=1) bp_one = esiclient.request(op_blueprint) if bp_one.status != 200: logger.error('Request failed [%s, %s, %d]: %s' % ( op_blueprint[0].url, op_blueprint[0].query, bp_one.status, bp_one.raw, )) self.end(TaskState.ERROR) return # prepare all other pages total_page = bp_one.header['X-Pages'][0] operations = [] for page in range(2, total_page + 1): operations.append( get_characters_blueprints(character_id=character_id, page=page)) # query all other pages and add the first page bp_list = esiclient.multi_request(operations) # parse the response and save everything for req, response in [(op_blueprint[0], bp_one)] + bp_list: for blueprint in response.data: original = (blueprint.quantity != -2) runs = blueprint.runs me = blueprint.material_efficiency te = blueprint.time_efficiency item_id = blueprint.type_id key = "%s-%d-%d-%d" % (item_id, original, me, te) if key not in blueprint_updated_list: blueprint_updated_list.add(key) if key not in blueprints: blueprints[key] = Blueprint( item_id=item_id, original=original, total_runs=runs, material_efficiency=me, time_efficiency=te, character_id=character_id, ) db.session.add(blueprints[key]) continue if not original: blueprints[key].total_runs += runs # delete every blueprint that have not been updated for key in (blueprint_init_list - blueprint_updated_list): db.session.delete(blueprints[key]) # update the token and the state token.request_try = 0 token.last_update = utcnow() token.cached_until = datetime( *parsedate(bp_one.header['Expires'][0])[:6]).replace(tzinfo=pytz.utc) db.session.commit() self.end(TaskState.SUCCESS)
def update_current_login_at(): if flask_login.current_user.is_authenticated: flask_login.current_user.current_login_at = utcnow() db.session.commit()