def _apply_override(self, key, val, overrides, path=None): if path is None: path = [] path.append(key) # if not self.isPriceValid(str(val)): if not self.isOverridePriceValid(str(val)): raise AppException(INVALID_OVERRIDE.format(val, key)) opr, price = self.overridePriceFromString(str(val)) if opr not in ('', '*', '/'): amount, short = self.priceFromString(price) tkey = self.cshorts[short] if tkey in overrides: self._apply_override(tkey, overrides.pop(tkey), overrides, path) if tkey in path and tkey != key: raise AppException( "Overrides contain a circular reference in path: {}". format(path)) # rate = self.convert(float(amount), short) rate = self.compilePrice(val, self.crates.get(key)) if rate <= 0: rate = 0 # raise AppException(INVALID_OVERRIDE_RATE.format(val, rate, key)) self.crates[key] = rate del path[-1]
def validateOverrides(self): for key, state in self.filter_state_overrides.items(): if not isinstance(state, bool): try: self.filter_state_overrides[key] = str2bool(str(state)) except ValueError: raise AppException( "Invalid state override '{}' for {}. Must be a boolean." .format(state, key)) if not cm.isOverridePriceValid(self.default_price_override): raise AppException("Invalid default price override {}".format( self.default_price_override)) for key, override in self.price_overrides.items(): if not cm.isOverridePriceValid(override): raise AppException("Invalid price override '{}' for {}".format( override, key)) if not cm.isOverridePriceValid(self.default_fprice_override): raise AppException( "Invalid default filter price override {}".format( self.default_fprice_override)) for key, override in self.filter_price_overrides.items(): if not cm.isOverridePriceValid(override): raise AppException( "Invalid filter price override '{}' for {}".format( override, key))
def validate(self): try: data = self.toDict() self.schema_validator.validate(data) except jsonschema.ValidationError as e: raise AppException(FILTER_VALIDATION_ERROR.format(get_verror_msg(e, data))) if self.criteria: for price in ('price_min', 'price_max'): if not self.baseId and price in self.criteria: if not cm.isOverridePriceValid(self.criteria[price]): raise AppException(Filter.FILTER_INVALID_PRICE.format(self.criteria[price], self.title)) if cm.isPriceRelative(self.criteria[price]): raise AppException(Filter.FILTER_INVALID_PRICE_BASE.format(self.criteria[price])) try: fgs = [FilterGroupFactory.create(FilterGroupType(fg['type']), fg) for fg in self.criteria.get('fgs', [])] for fg in fgs: for mf in fg.mfs: if mf.type != ModFilterType.Pseudo: re.compile(mf.expr) except re.error as e: raise AppException(Filter.FILTER_INVALID_REGEX.format(e.pattern, self.title, e))
def update(self, force_update=False, accept_empty=False): if not force_update and not self.needUpdate: return # print('updating currency..') try: shorts = self.shorts rates = {} def get_count(currency): return max( currency['receive']['count'] if currency['receive'] else 0, currency['pay']['count'] if currency['pay'] else 0) for url in CurrencyManager.CURRENCY_API: data = getJsonFromURL(url.format(config.league)) if data is None and not accept_empty: raise AppException( "Currency update failed. Empty response from server.") if data: # shorts.update({currency['name']: currency['shorthands'] for currency in data["currencyDetails"]}) rates.update({ currency['currencyTypeName']: float(currency['chaosEquivalent']) for currency in data["lines"] if get_count(currency) >= self.confidence_level }) # cur_shorts = dict(self.shorts) # for name in cur_shorts: # shorts[name] = list(set(cur_shorts[name] + shorts.get(name, []))) # can use update if we want to keep information from past updates, more robust if server returns less data # dict(self.rates).update(rates) self.compile(shorts, rates, last_update=datetime.utcnow() if rates else None) except pycurl.error as e: raise AppException( "Currency update failed. Connection error: {}".format(e)) except AppException: raise except (KeyError, ValueError) as e: raise AppException( "Currency update failed. Parsing error: {}".format(e)) except Exception as e: logexception() raise AppException( "Currency update failed. Unexpected error: {}".format(e))
def loadFiltersFromFile(cls, fname, validate_data): filters = [] cur_fname = fname try: with cls.filter_file_lock[fname]: with open(cur_fname, encoding="utf-8", errors="replace") as f: data = json.load(f) cur_fname = FILTERS_FILE_SCHEMA_FNAME # TODO: move schema loading to main init and store in class with open(cur_fname) as f: schema = json.load(f) # normalize keys and values data = lower_json(data) jsonschema.validate(data, schema) for item in data.get('filters', []): fltr = Filter.fromDict(item) filters.append(fltr) ver = data.get('version', FilterVersion.V1) if ver != FilterVersion.Latest: FilterManager.convert(ver, filters) last_update = data.get('last_update', '') if validate_data: for fltr in filters: fltr.validate() try: last_update = datetime.strptime(last_update, '%Y-%m-%dT%H:%M:%S.%f') except ValueError: last_update = datetime.utcnow() - timedelta( minutes=FilterManager.UPDATE_INTERVAL) except FileNotFoundError: if cur_fname != FILTERS_FILE_SCHEMA_FNAME: raise else: raise AppException(FILTER_FILE_MISSING.format(cur_fname)) except jsonschema.ValidationError as e: raise AppException( FILTERS_FILE_VALIDATION_ERROR.format(get_verror_msg(e, data))) except jsonschema.SchemaError as e: raise AppException(FILTERS_FILE_SCHEMA_ERROR.format(e.message)) except json.decoder.JSONDecodeError as e: raise AppException(FILTER_INVALID_JSON.format(e, cur_fname)) return filters, last_update
def validateConfig(self): if not cm.isPriceValid(self.price_threshold): raise AppException("Invalid price threshold {}".format( self.price_threshold)) if self.budget and not cm.isPriceValid(self.budget): raise AppException("Invalid budget price {}".format(self.budget)) if self.default_min_price and not cm.isPriceValid( self.default_min_price): raise AppException("Invalid minimum price {}".format( self.default_min_price)) self.validateOverrides()
def init(cls): try: with open(Filter.FILTER_SCHEMA_FNAME) as f: schema = json.load(f) jsonschema.validate({}, schema) except jsonschema.ValidationError: cls.schema_validator = jsonschema.Draft4Validator(schema) except jsonschema.SchemaError as e: raise AppException('Failed loading filter validation schema.\n{}'.format(FILTER_SCHEMA_ERROR.format(e))) # except FileNotFoundError as e: except Exception as e: raise AppException('Failed loading filter validation schema.\n{}\n' 'Make sure the file are valid and in place.'.format(e))
def compileFilters(self, force_validation=False): with self.compile_lock: try: if self.validation_required or force_validation: start = time.time() valid = self.validateFilters() end = time.time() - start msgr.send_msg( 'Filters validation time {:.4f}s'.format(end), logging.DEBUG) if not valid: raise AppException('Filter validation failed.') self._compileFilters() msg = 'Filters compiled successfully.' if len(self.getActiveFilters()): msg += ' {} are active.'.format( len(self.getActiveFilters())) msgr.send_msg(msg, logging.INFO) except Exception as e: # configuration is valid yet compile failed, stop self.compiledFilters = [] self.activeFilters = [] self.compiled_item_prices = {} self.compiled_filter_prices = {} if isinstance(e, AppException): msgr.send_msg(e, logging.ERROR) else: logexception() msgr.send_msg( 'Unexpected error while compiling filters: {}'.format( e), logging.ERROR) finally: msgr.send_object(FiltersInfo())
def loadAutoFilters(self, validate=True): try: self.autoFilters, self.last_update = FilterManager.loadFiltersFromFile( _AUTO_FILTERS_FNAME, validate) self.item_prices = self.getPrices(self.autoFilters) except FileNotFoundError as e: raise AppException( "Loading generated filters failed. Missing file {}", e.filename) except AppException: raise except Exception as e: logexception() raise AppException( "Loading generated filters failed. Unexpected error: {}". format(e))
def _get_latest_id(self, is_beta): latest_id = None failed_attempts = 0 sleep_time = 0 if is_beta: ninja_api_nextid_field = 'next_beta_change_id' else: ninja_api_nextid_field = 'next_change_id' while not self._stop.wait(sleep_time) and not latest_id: try: data = getJsonFromURL(NINJA_API) if data is None: msgr.send_msg("Error retrieving latest id from API, bad response", logging.WARN) elif ninja_api_nextid_field not in data: raise AppException( "Error retrieving latest id from API, missing {} key".format(ninja_api_nextid_field)) else: latest_id = data[ninja_api_nextid_field] break except pycurl.error as e: errno, msg = e.args msgr.send_tmsg("Connection error {}: {}".format(errno, msg), logging.WARN) finally: failed_attempts += 1 sleep_time = min(2 ** failed_attempts, 30) return latest_id
def loadConfig(self): try: try: with self.config_file_lock: with open(FILTERS_CFG_FNAME, encoding="utf-8", errors="replace") as f: data = json.load(f) except FileNotFoundError: data = {} self.disabled_categories = data.get('disabled_categories', []) self.price_threshold = data.get('price_threshold', self.DEFAULT_PRICE_THRESHOLD) self.budget = data.get('budget', self.DEFAULT_BUDGET) self.default_min_price = data.get('default_min_price', self.DEFAULT_MIN_PRICE) self.default_price_override = data.get('default_price_override', self.DEFAULT_PRICE_OVERRIDE) self.default_fprice_override = data.get( 'default_fprice_override', self.DEFAULT_FPRICE_OVERRIDE) self.price_overrides = data.get('price_overrides', {}) self.filter_price_overrides = data.get('filter_price_overrides', {}) self.filter_state_overrides = data.get('filter_state_overrides', {}) self.confidence_level = data.get('confidence_level', self.DEFAULT_CONFIDENCE_LEVEL) self.enable_5l_filters = data.get('enable_5l_filters', self.DEFAULT_ENABLE_5L_FILTERS) try: self.validateConfig() except AppException as e: raise AppException( 'Failed validating filters configuration. {}'.format(e)) self.saveConfig() except Exception as e: logexception() raise AppException( 'Failed loading filters configuration. Unexpected error: {}'. format(e))
def loadUserFilters(self, validate=True): try: self.userFilters, last_update = FilterManager.loadFiltersFromFile( _USER_FILTERS_FNAME, validate) except FileNotFoundError: self._loadDefaultFilters() self.saveUserFilters() except AppException: raise except Exception as e: logexception() raise AppException( "Loading user filters failed. Unexpected error: {}".format(e))
def init(cls): try: with open(cls.BASE_TYPES_FNAME) as f: data = json.load(f) cls.base_types = data cls.base_type_to_id = { base_type: class_id for class_id in data for base_type in data[class_id] } except Exception as e: raise AppException( 'Failed loading item base types.\n{}\n' 'Make sure the file are valid and in place.'.format(e))
def load_base(self): try: with open(CurrencyManager.CURRENCY_BASE_FNAME, encoding="utf-8", errors="replace") as f: data = json.load(f) self.shorts = data.get('shorts', {}) self.whisper = data.get('whisper', {}) for curr in self.shorts: self.shorts[curr] = list( set([short.lower() for short in self.shorts[curr]])) except FileNotFoundError as e: raise AppException( 'Loading currency base failed. Missing file {}'.format( e.filename))
def compileFilter(self, fltr, path=None): if path is None: path = [] if fltr.id in path: raise AppException( "Circular reference detected while compiling filters: {}". format(path)) path.append(fltr.id) if not fltr.baseId or fltr.baseId == fltr.id: baseComp = {} else: baseFilter = self.getFilterById( fltr.baseId, itertools.chain(self.userFilters, self.autoFilters)) if baseFilter is None: # try using last compilation compiledFilter = self.getFilterById( fltr.baseId, self.activeFilters, lambda x, y: x.fltr.id == y) if compiledFilter is None: raise CompileException( "Base filter '{}' not found.".format(fltr.baseId)) # return None baseComp = self.compileFilter(compiledFilter.fltr, path) else: baseComp = self.compileFilter(baseFilter, path) # if baseComp is None: # return None comp = fltr.compile(baseComp) if fltr.id.startswith('_'): val_override = self.price_overrides.get( fltr.id, self.default_price_override) comp['price_max'] = cm.compilePrice(val_override, comp['price_max']) # return fltr.compile(baseComp) return comp
def scan(self): msgr.send_msg("Scan initializing..") os.makedirs('tmp', exist_ok=True) os.makedirs('log', exist_ok=True) is_beta = config.league.lower().startswith('beta ') if is_beta: self.poe_api_url = POE_BETA_API self.league = re.sub('beta ', '', config.league, flags=re.IGNORECASE) else: self.poe_api_url = POE_API self.league = config.league # assertions if not cm.initialized: raise AppException("Currency information must be initialized before starting a scan.") if not fm.initialized: raise AppException("Filters information must be initialized before starting a scan.") if cm.needUpdate: try: cm.update() msgr.send_msg("Currency rates updated successfully.") except AppException as e: msgr.send_msg(e, logging.ERROR) if cm.initialized: msgr.send_msg('Using currency information from a local copy..', logging.WARN) if fm.needUpdate: try: msgr.send_msg("Generating filters from API..") fm.fetchFromAPI() except AppException as e: # filterFallback = True msgr.send_msg(e, logging.ERROR) msgr.send_msg('Compiling filters..', logging.INFO) fm.compileFilters(force_validation=True) filters = fm.getActiveFilters() if not len(filters): raise AppException("No filters are active. Stopping..") self.stateMgr.loadState() if self.stateMgr.getChangeId() == "" or str(config.scan_mode).lower() == "latest": msgr.send_msg("Fetching latest id from API..") latest_id = self._get_latest_id(is_beta) if latest_id: if not self.stateMgr.getChangeId() or get_delta(self.stateMgr.getChangeId(), latest_id) > 0: self.stateMgr.saveState(latest_id) else: msgr.send_msg('Saved ID is more recent, continuing..') elif not self._stop.is_set(): raise AppException("Failed retrieving latest ID from API") self.updater.start() self.notifier.start() get_next = True msgr.send_msg("Scanning started") msgr.send_update_id(self.stateMgr.getChangeId()) while not self._stop.is_set(): if self.downloader is None or not self.downloader.is_alive(): if self.downloader: msgr.send_msg("Download thread ended abruptly. Restarting it..", logging.WARN) self.downloader = Downloader(self.stateMgr.getChangeId(), conns=config.max_conns) self.downloader.start() if self.parser is None or not self.parser.is_alive() and not self.parser.signal_stop: if self.parser: msgr.send_msg("Parser thread ended abruptly. Restarting it..", logging.WARN) if config.num_workers > 0: workers = config.num_workers else: workers = max((os.cpu_count() or 1) - 1, 1) self.parser = ParserThread(workers, self.league, self.stateMgr, self.handleResult) self.parser.start() try: if get_next: req_id, resp = self.downloader.get(timeout=0.5) get_next = False self.parser.put(req_id, resp, timeout=0.5) get_next = True except Full: msgr.send_msg("Parser queue is full.. waiting for parser..", logging.WARN) except Empty: continue
def init(self): try: self.load() except Exception as e: raise AppException( 'Failed to load item mods information.\n{}'.format(e))
def fetchFromAPI(self, force_update=False, accept_empty=False): if not force_update and not self.needUpdate: return # print('updating filters..') try: filter_ids = [] filters = [] def name_to_id(name): return '_' + name.lower().replace(' ', '_') def get_unique_id(title, name, category, links): title_id = name_to_id(title) if title_id not in filter_ids: return title_id name_id = name_to_id('{}{}'.format( name, ' {}L'.format(links) if links else '')) if name_id not in filter_ids: # print('id {} was taken, using name id {} instead'.format(title_id, name_id)) return name_id category_id = name_to_id(title + ' ' + category) if category_id not in filter_ids: # print('id {} was taken, using category id {} instead'.format(title_id, category_id)) return category_id id = title_id n = 2 while id in filter_ids: id = '{}{}'.format(title_id, n) n += 1 # if n > 2: # print('id {} was taken, using {} instead'.format(title_id, id)) return id c = pycurl.Curl() for url in _URLS: furl = url.format(config.league) data = getJsonFromURL(furl, handle=c, max_attempts=3) if data is None and not accept_empty: raise AppException( "Filters update failed. Empty response from server") if data: category = re.match(".*Get(.*)Overview", furl).group(1).lower() for item in data['lines']: if item['count'] < self.confidence_level: continue priority = FilterPriority.AutoBase crit = {} # crit['price_max'] = "{} exalted".format(float(item.get('exaltedValue', 0))) crit['price_max'] = "{} chaos".format( float(item.get('chaosValue', 0))) base = item['baseType'] if category not in ( 'essence', ) else None name = item['name'] if base: name += ' ' + base crit['name'] = ['"{}"'.format(name)] try: rarity = ItemRarity(item['itemClass']) crit['rarity'] = [_ITEM_TYPE[rarity]] except ValueError: rarity = None crit['buyout'] = True if category in ('uniquearmour', 'uniqueweapon'): crit['corrupted'] = False links = item['links'] title = "{} {} {}".format( 'Legacy' if rarity == ItemRarity.Relic else '', item['name'], item['variant'] if item['variant'] is not None else '').strip() if links: title = '{} {}L'.format(title, links) crit['links_min'] = links if links == 5: priority += 1 elif links == 6: priority += 2 tier = item['mapTier'] if tier: crit['level_min'] = tier crit['level_max'] = tier id = get_unique_id(title, name, category, links) filter_ids.append(id) fltr = Filter(title, crit, False, category, id=id, priority=priority) if item['variant'] is not None: if item['variant'] not in _VARIANTS: msgr.send_msg( "Unknown variant {} in item {}".format( item['variant'], item['name']), logging.WARN) else: # crit['explicit'] = {'mods': [{'expr': _VARIANTS[item['variant']]}]} mfs = _VARIANTS[item['variant']] if mfs: fg = AllFilterGroup() for expr in _VARIANTS[item['variant']]: fg.addModFilter( ModFilter(ModFilterType.Explicit, expr)) fltr.criteria['fgs'] = [fg.toDict()] fltr.validate() filters.append(fltr) self.autoFilters = filters self.item_prices = self.getPrices(self.autoFilters) self.saveAutoFilters() self.last_update = datetime.utcnow() if filters else None except pycurl.error as e: raise AppException( "Filters update failed. Connection error: {}".format(e)) except (KeyError, ValueError) as e: raise AppException( "Filters update failed. Parsing error: {}".format(e)) except AppException: raise except Exception as e: logexception() raise AppException( "Filters update failed. Unexpected error: {}".format(e))