def _apply_override(self, key, val, overrides, path=None):
if path is None:
path = []
path.append(key)
# if not self.isPriceValid(str(val)):
if not self.isOverridePriceValid(str(val)):
raise AppException(INVALID_OVERRIDE.format(val, key))
opr, price = self.overridePriceFromString(str(val))
if opr not in ('', '*', '/'):
amount, short = self.priceFromString(price)
tkey = self.cshorts[short]
if tkey in overrides:
self._apply_override(tkey, overrides.pop(tkey), overrides,
path)
if tkey in path and tkey != key:
raise AppException(
"Overrides contain a circular reference in path: {}".
format(path))
# rate = self.convert(float(amount), short)
rate = self.compilePrice(val, self.crates.get(key))
if rate <= 0:
rate = 0
# raise AppException(INVALID_OVERRIDE_RATE.format(val, rate, key))
self.crates[key] = rate
del path[-1]
def validateOverrides(self):
for key, state in self.filter_state_overrides.items():
if not isinstance(state, bool):
try:
self.filter_state_overrides[key] = str2bool(str(state))
except ValueError:
raise AppException(
"Invalid state override '{}' for {}. Must be a boolean."
.format(state, key))
if not cm.isOverridePriceValid(self.default_price_override):
raise AppException("Invalid default price override {}".format(
self.default_price_override))
for key, override in self.price_overrides.items():
if not cm.isOverridePriceValid(override):
raise AppException("Invalid price override '{}' for {}".format(
override, key))
if not cm.isOverridePriceValid(self.default_fprice_override):
raise AppException(
"Invalid default filter price override {}".format(
self.default_fprice_override))
for key, override in self.filter_price_overrides.items():
if not cm.isOverridePriceValid(override):
raise AppException(
"Invalid filter price override '{}' for {}".format(
override, key))
def validate(self):
try:
data = self.toDict()
self.schema_validator.validate(data)
except jsonschema.ValidationError as e:
raise AppException(FILTER_VALIDATION_ERROR.format(get_verror_msg(e, data)))
if self.criteria:
for price in ('price_min', 'price_max'):
if not self.baseId and price in self.criteria:
if not cm.isOverridePriceValid(self.criteria[price]):
raise AppException(Filter.FILTER_INVALID_PRICE.format(self.criteria[price], self.title))
if cm.isPriceRelative(self.criteria[price]):
raise AppException(Filter.FILTER_INVALID_PRICE_BASE.format(self.criteria[price]))
try:
fgs = [FilterGroupFactory.create(FilterGroupType(fg['type']), fg) for fg in self.criteria.get('fgs', [])]
for fg in fgs:
for mf in fg.mfs:
if mf.type != ModFilterType.Pseudo:
re.compile(mf.expr)
except re.error as e:
raise AppException(Filter.FILTER_INVALID_REGEX.format(e.pattern, self.title, e))
def update(self, force_update=False, accept_empty=False):
if not force_update and not self.needUpdate:
return
# print('updating currency..')
try:
shorts = self.shorts
rates = {}
def get_count(currency):
return max(
currency['receive']['count'] if currency['receive'] else 0,
currency['pay']['count'] if currency['pay'] else 0)
for url in CurrencyManager.CURRENCY_API:
data = getJsonFromURL(url.format(config.league))
if data is None and not accept_empty:
raise AppException(
"Currency update failed. Empty response from server.")
if data:
# shorts.update({currency['name']: currency['shorthands'] for currency in data["currencyDetails"]})
rates.update({
currency['currencyTypeName']:
float(currency['chaosEquivalent'])
for currency in data["lines"]
if get_count(currency) >= self.confidence_level
})
# cur_shorts = dict(self.shorts)
# for name in cur_shorts:
# shorts[name] = list(set(cur_shorts[name] + shorts.get(name, [])))
# can use update if we want to keep information from past updates, more robust if server returns less data
# dict(self.rates).update(rates)
self.compile(shorts,
rates,
last_update=datetime.utcnow() if rates else None)
except pycurl.error as e:
raise AppException(
"Currency update failed. Connection error: {}".format(e))
except AppException:
raise
except (KeyError, ValueError) as e:
raise AppException(
"Currency update failed. Parsing error: {}".format(e))
except Exception as e:
logexception()
raise AppException(
"Currency update failed. Unexpected error: {}".format(e))
def loadFiltersFromFile(cls, fname, validate_data):
filters = []
cur_fname = fname
try:
with cls.filter_file_lock[fname]:
with open(cur_fname, encoding="utf-8", errors="replace") as f:
data = json.load(f)
cur_fname = FILTERS_FILE_SCHEMA_FNAME # TODO: move schema loading to main init and store in class
with open(cur_fname) as f:
schema = json.load(f)
# normalize keys and values
data = lower_json(data)
jsonschema.validate(data, schema)
for item in data.get('filters', []):
fltr = Filter.fromDict(item)
filters.append(fltr)
ver = data.get('version', FilterVersion.V1)
if ver != FilterVersion.Latest:
FilterManager.convert(ver, filters)
last_update = data.get('last_update', '')
if validate_data:
for fltr in filters:
fltr.validate()
try:
last_update = datetime.strptime(last_update,
'%Y-%m-%dT%H:%M:%S.%f')
except ValueError:
last_update = datetime.utcnow() - timedelta(
minutes=FilterManager.UPDATE_INTERVAL)
except FileNotFoundError:
if cur_fname != FILTERS_FILE_SCHEMA_FNAME:
raise
else:
raise AppException(FILTER_FILE_MISSING.format(cur_fname))
except jsonschema.ValidationError as e:
raise AppException(
FILTERS_FILE_VALIDATION_ERROR.format(get_verror_msg(e, data)))
except jsonschema.SchemaError as e:
raise AppException(FILTERS_FILE_SCHEMA_ERROR.format(e.message))
except json.decoder.JSONDecodeError as e:
raise AppException(FILTER_INVALID_JSON.format(e, cur_fname))
return filters, last_update
def validateConfig(self):
if not cm.isPriceValid(self.price_threshold):
raise AppException("Invalid price threshold {}".format(
self.price_threshold))
if self.budget and not cm.isPriceValid(self.budget):
raise AppException("Invalid budget price {}".format(self.budget))
if self.default_min_price and not cm.isPriceValid(
self.default_min_price):
raise AppException("Invalid minimum price {}".format(
self.default_min_price))
self.validateOverrides()
def init(cls):
try:
with open(Filter.FILTER_SCHEMA_FNAME) as f:
schema = json.load(f)
jsonschema.validate({}, schema)
except jsonschema.ValidationError:
cls.schema_validator = jsonschema.Draft4Validator(schema)
except jsonschema.SchemaError as e:
raise AppException('Failed loading filter validation schema.\n{}'.format(FILTER_SCHEMA_ERROR.format(e)))
# except FileNotFoundError as e:
except Exception as e:
raise AppException('Failed loading filter validation schema.\n{}\n'
'Make sure the file are valid and in place.'.format(e))
def compileFilters(self, force_validation=False):
with self.compile_lock:
try:
if self.validation_required or force_validation:
start = time.time()
valid = self.validateFilters()
end = time.time() - start
msgr.send_msg(
'Filters validation time {:.4f}s'.format(end),
logging.DEBUG)
if not valid:
raise AppException('Filter validation failed.')
self._compileFilters()
msg = 'Filters compiled successfully.'
if len(self.getActiveFilters()):
msg += ' {} are active.'.format(
len(self.getActiveFilters()))
msgr.send_msg(msg, logging.INFO)
except Exception as e:
# configuration is valid yet compile failed, stop
self.compiledFilters = []
self.activeFilters = []
self.compiled_item_prices = {}
self.compiled_filter_prices = {}
if isinstance(e, AppException):
msgr.send_msg(e, logging.ERROR)
else:
logexception()
msgr.send_msg(
'Unexpected error while compiling filters: {}'.format(
e), logging.ERROR)
finally:
msgr.send_object(FiltersInfo())
def loadAutoFilters(self, validate=True):
try:
self.autoFilters, self.last_update = FilterManager.loadFiltersFromFile(
_AUTO_FILTERS_FNAME, validate)
self.item_prices = self.getPrices(self.autoFilters)
except FileNotFoundError as e:
raise AppException(
"Loading generated filters failed. Missing file {}",
e.filename)
except AppException:
raise
except Exception as e:
logexception()
raise AppException(
"Loading generated filters failed. Unexpected error: {}".
format(e))
def _get_latest_id(self, is_beta):
latest_id = None
failed_attempts = 0
sleep_time = 0
if is_beta:
ninja_api_nextid_field = 'next_beta_change_id'
else:
ninja_api_nextid_field = 'next_change_id'
while not self._stop.wait(sleep_time) and not latest_id:
try:
data = getJsonFromURL(NINJA_API)
if data is None:
msgr.send_msg("Error retrieving latest id from API, bad response", logging.WARN)
elif ninja_api_nextid_field not in data:
raise AppException(
"Error retrieving latest id from API, missing {} key".format(ninja_api_nextid_field))
else:
latest_id = data[ninja_api_nextid_field]
break
except pycurl.error as e:
errno, msg = e.args
msgr.send_tmsg("Connection error {}: {}".format(errno, msg), logging.WARN)
finally:
failed_attempts += 1
sleep_time = min(2 ** failed_attempts, 30)
return latest_id
def loadConfig(self):
try:
try:
with self.config_file_lock:
with open(FILTERS_CFG_FNAME,
encoding="utf-8",
errors="replace") as f:
data = json.load(f)
except FileNotFoundError:
data = {}
self.disabled_categories = data.get('disabled_categories', [])
self.price_threshold = data.get('price_threshold',
self.DEFAULT_PRICE_THRESHOLD)
self.budget = data.get('budget', self.DEFAULT_BUDGET)
self.default_min_price = data.get('default_min_price',
self.DEFAULT_MIN_PRICE)
self.default_price_override = data.get('default_price_override',
self.DEFAULT_PRICE_OVERRIDE)
self.default_fprice_override = data.get(
'default_fprice_override', self.DEFAULT_FPRICE_OVERRIDE)
self.price_overrides = data.get('price_overrides', {})
self.filter_price_overrides = data.get('filter_price_overrides',
{})
self.filter_state_overrides = data.get('filter_state_overrides',
{})
self.confidence_level = data.get('confidence_level',
self.DEFAULT_CONFIDENCE_LEVEL)
self.enable_5l_filters = data.get('enable_5l_filters',
self.DEFAULT_ENABLE_5L_FILTERS)
try:
self.validateConfig()
except AppException as e:
raise AppException(
'Failed validating filters configuration. {}'.format(e))
self.saveConfig()
except Exception as e:
logexception()
raise AppException(
'Failed loading filters configuration. Unexpected error: {}'.
format(e))
def loadUserFilters(self, validate=True):
try:
self.userFilters, last_update = FilterManager.loadFiltersFromFile(
_USER_FILTERS_FNAME, validate)
except FileNotFoundError:
self._loadDefaultFilters()
self.saveUserFilters()
except AppException:
raise
except Exception as e:
logexception()
raise AppException(
"Loading user filters failed. Unexpected error: {}".format(e))
def init(cls):
try:
with open(cls.BASE_TYPES_FNAME) as f:
data = json.load(f)
cls.base_types = data
cls.base_type_to_id = {
base_type: class_id
for class_id in data for base_type in data[class_id]
}
except Exception as e:
raise AppException(
'Failed loading item base types.\n{}\n'
'Make sure the file are valid and in place.'.format(e))
def load_base(self):
try:
with open(CurrencyManager.CURRENCY_BASE_FNAME,
encoding="utf-8",
errors="replace") as f:
data = json.load(f)
self.shorts = data.get('shorts', {})
self.whisper = data.get('whisper', {})
for curr in self.shorts:
self.shorts[curr] = list(
set([short.lower() for short in self.shorts[curr]]))
except FileNotFoundError as e:
raise AppException(
'Loading currency base failed. Missing file {}'.format(
e.filename))
def compileFilter(self, fltr, path=None):
if path is None:
path = []
if fltr.id in path:
raise AppException(
"Circular reference detected while compiling filters: {}".
format(path))
path.append(fltr.id)
if not fltr.baseId or fltr.baseId == fltr.id:
baseComp = {}
else:
baseFilter = self.getFilterById(
fltr.baseId, itertools.chain(self.userFilters,
self.autoFilters))
if baseFilter is None:
# try using last compilation
compiledFilter = self.getFilterById(
fltr.baseId, self.activeFilters,
lambda x, y: x.fltr.id == y)
if compiledFilter is None:
raise CompileException(
"Base filter '{}' not found.".format(fltr.baseId))
# return None
baseComp = self.compileFilter(compiledFilter.fltr, path)
else:
baseComp = self.compileFilter(baseFilter, path)
# if baseComp is None:
# return None
comp = fltr.compile(baseComp)
if fltr.id.startswith('_'):
val_override = self.price_overrides.get(
fltr.id, self.default_price_override)
comp['price_max'] = cm.compilePrice(val_override,
comp['price_max'])
# return fltr.compile(baseComp)
return comp
def scan(self):
msgr.send_msg("Scan initializing..")
os.makedirs('tmp', exist_ok=True)
os.makedirs('log', exist_ok=True)
is_beta = config.league.lower().startswith('beta ')
if is_beta:
self.poe_api_url = POE_BETA_API
self.league = re.sub('beta ', '', config.league, flags=re.IGNORECASE)
else:
self.poe_api_url = POE_API
self.league = config.league
# assertions
if not cm.initialized:
raise AppException("Currency information must be initialized before starting a scan.")
if not fm.initialized:
raise AppException("Filters information must be initialized before starting a scan.")
if cm.needUpdate:
try:
cm.update()
msgr.send_msg("Currency rates updated successfully.")
except AppException as e:
msgr.send_msg(e, logging.ERROR)
if cm.initialized:
msgr.send_msg('Using currency information from a local copy..', logging.WARN)
if fm.needUpdate:
try:
msgr.send_msg("Generating filters from API..")
fm.fetchFromAPI()
except AppException as e:
# filterFallback = True
msgr.send_msg(e, logging.ERROR)
msgr.send_msg('Compiling filters..', logging.INFO)
fm.compileFilters(force_validation=True)
filters = fm.getActiveFilters()
if not len(filters):
raise AppException("No filters are active. Stopping..")
self.stateMgr.loadState()
if self.stateMgr.getChangeId() == "" or str(config.scan_mode).lower() == "latest":
msgr.send_msg("Fetching latest id from API..")
latest_id = self._get_latest_id(is_beta)
if latest_id:
if not self.stateMgr.getChangeId() or get_delta(self.stateMgr.getChangeId(), latest_id) > 0:
self.stateMgr.saveState(latest_id)
else:
msgr.send_msg('Saved ID is more recent, continuing..')
elif not self._stop.is_set():
raise AppException("Failed retrieving latest ID from API")
self.updater.start()
self.notifier.start()
get_next = True
msgr.send_msg("Scanning started")
msgr.send_update_id(self.stateMgr.getChangeId())
while not self._stop.is_set():
if self.downloader is None or not self.downloader.is_alive():
if self.downloader:
msgr.send_msg("Download thread ended abruptly. Restarting it..", logging.WARN)
self.downloader = Downloader(self.stateMgr.getChangeId(), conns=config.max_conns)
self.downloader.start()
if self.parser is None or not self.parser.is_alive() and not self.parser.signal_stop:
if self.parser:
msgr.send_msg("Parser thread ended abruptly. Restarting it..", logging.WARN)
if config.num_workers > 0:
workers = config.num_workers
else:
workers = max((os.cpu_count() or 1) - 1, 1)
self.parser = ParserThread(workers, self.league, self.stateMgr, self.handleResult)
self.parser.start()
try:
if get_next:
req_id, resp = self.downloader.get(timeout=0.5)
get_next = False
self.parser.put(req_id, resp, timeout=0.5)
get_next = True
except Full:
msgr.send_msg("Parser queue is full.. waiting for parser..", logging.WARN)
except Empty:
continue
def init(self):
try:
self.load()
except Exception as e:
raise AppException(
'Failed to load item mods information.\n{}'.format(e))
def fetchFromAPI(self, force_update=False, accept_empty=False):
if not force_update and not self.needUpdate:
return
# print('updating filters..')
try:
filter_ids = []
filters = []
def name_to_id(name):
return '_' + name.lower().replace(' ', '_')
def get_unique_id(title, name, category, links):
title_id = name_to_id(title)
if title_id not in filter_ids:
return title_id
name_id = name_to_id('{}{}'.format(
name, ' {}L'.format(links) if links else ''))
if name_id not in filter_ids:
# print('id {} was taken, using name id {} instead'.format(title_id, name_id))
return name_id
category_id = name_to_id(title + ' ' + category)
if category_id not in filter_ids:
# print('id {} was taken, using category id {} instead'.format(title_id, category_id))
return category_id
id = title_id
n = 2
while id in filter_ids:
id = '{}{}'.format(title_id, n)
n += 1
# if n > 2:
# print('id {} was taken, using {} instead'.format(title_id, id))
return id
c = pycurl.Curl()
for url in _URLS:
furl = url.format(config.league)
data = getJsonFromURL(furl, handle=c, max_attempts=3)
if data is None and not accept_empty:
raise AppException(
"Filters update failed. Empty response from server")
if data:
category = re.match(".*Get(.*)Overview",
furl).group(1).lower()
for item in data['lines']:
if item['count'] < self.confidence_level:
continue
priority = FilterPriority.AutoBase
crit = {}
# crit['price_max'] = "{} exalted".format(float(item.get('exaltedValue', 0)))
crit['price_max'] = "{} chaos".format(
float(item.get('chaosValue', 0)))
base = item['baseType'] if category not in (
'essence', ) else None
name = item['name']
if base:
name += ' ' + base
crit['name'] = ['"{}"'.format(name)]
try:
rarity = ItemRarity(item['itemClass'])
crit['rarity'] = [_ITEM_TYPE[rarity]]
except ValueError:
rarity = None
crit['buyout'] = True
if category in ('uniquearmour', 'uniqueweapon'):
crit['corrupted'] = False
links = item['links']
title = "{} {} {}".format(
'Legacy' if rarity == ItemRarity.Relic else '',
item['name'], item['variant']
if item['variant'] is not None else '').strip()
if links:
title = '{} {}L'.format(title, links)
crit['links_min'] = links
if links == 5:
priority += 1
elif links == 6:
priority += 2
tier = item['mapTier']
if tier:
crit['level_min'] = tier
crit['level_max'] = tier
id = get_unique_id(title, name, category, links)
filter_ids.append(id)
fltr = Filter(title,
crit,
False,
category,
id=id,
priority=priority)
if item['variant'] is not None:
if item['variant'] not in _VARIANTS:
msgr.send_msg(
"Unknown variant {} in item {}".format(
item['variant'], item['name']),
logging.WARN)
else:
# crit['explicit'] = {'mods': [{'expr': _VARIANTS[item['variant']]}]}
mfs = _VARIANTS[item['variant']]
if mfs:
fg = AllFilterGroup()
for expr in _VARIANTS[item['variant']]:
fg.addModFilter(
ModFilter(ModFilterType.Explicit,
expr))
fltr.criteria['fgs'] = [fg.toDict()]
fltr.validate()
filters.append(fltr)
self.autoFilters = filters
self.item_prices = self.getPrices(self.autoFilters)
self.saveAutoFilters()
self.last_update = datetime.utcnow() if filters else None
except pycurl.error as e:
raise AppException(
"Filters update failed. Connection error: {}".format(e))
except (KeyError, ValueError) as e:
raise AppException(
"Filters update failed. Parsing error: {}".format(e))
except AppException:
raise
except Exception as e:
logexception()
raise AppException(
"Filters update failed. Unexpected error: {}".format(e))
