def loadAutoFilters(self, validate=True):
try:
self.autoFilters, self.last_update = FilterManager.loadFiltersFromFile(
_AUTO_FILTERS_FNAME, validate)
self.item_prices = self.getPrices(self.autoFilters)
except FileNotFoundError as e:
raise AppException(
"Loading generated filters failed. Missing file {}",
e.filename)
except AppException:
raise
except Exception as e:
logexception()
raise AppException(
"Loading generated filters failed. Unexpected error: {}".
format(e))
def loadConfig(self):
try:
try:
with self.config_file_lock:
with open(FILTERS_CFG_FNAME,
encoding="utf-8",
errors="replace") as f:
data = json.load(f)
except FileNotFoundError:
data = {}
self.disabled_categories = data.get('disabled_categories', [])
self.price_threshold = data.get('price_threshold',
self.DEFAULT_PRICE_THRESHOLD)
self.budget = data.get('budget', self.DEFAULT_BUDGET)
self.default_min_price = data.get('default_min_price',
self.DEFAULT_MIN_PRICE)
self.default_price_override = data.get('default_price_override',
self.DEFAULT_PRICE_OVERRIDE)
self.default_fprice_override = data.get(
'default_fprice_override', self.DEFAULT_FPRICE_OVERRIDE)
self.price_overrides = data.get('price_overrides', {})
self.filter_price_overrides = data.get('filter_price_overrides',
{})
self.filter_state_overrides = data.get('filter_state_overrides',
{})
self.confidence_level = data.get('confidence_level',
self.DEFAULT_CONFIDENCE_LEVEL)
self.enable_5l_filters = data.get('enable_5l_filters',
self.DEFAULT_ENABLE_5L_FILTERS)
try:
self.validateConfig()
except AppException as e:
raise AppException(
'Failed validating filters configuration. {}'.format(e))
self.saveConfig()
except Exception as e:
logexception()
raise AppException(
'Failed loading filters configuration. Unexpected error: {}'.
format(e))
def loadUserFilters(self, validate=True):
try:
self.userFilters, last_update = FilterManager.loadFiltersFromFile(
_USER_FILTERS_FNAME, validate)
except FileNotFoundError:
self._loadDefaultFilters()
self.saveUserFilters()
except AppException:
raise
except Exception as e:
logexception()
raise AppException(
"Loading user filters failed. Unexpected error: {}".format(e))
def init(cls):
try:
with open(cls.BASE_TYPES_FNAME) as f:
data = json.load(f)
cls.base_types = data
cls.base_type_to_id = {
base_type: class_id
for class_id in data for base_type in data[class_id]
}
except Exception as e:
raise AppException(
'Failed loading item base types.\n{}\n'
'Make sure the file are valid and in place.'.format(e))
def load_base(self):
try:
with open(CurrencyManager.CURRENCY_BASE_FNAME,
encoding="utf-8",
errors="replace") as f:
data = json.load(f)
self.shorts = data.get('shorts', {})
self.whisper = data.get('whisper', {})
for curr in self.shorts:
self.shorts[curr] = list(
set([short.lower() for short in self.shorts[curr]]))
except FileNotFoundError as e:
raise AppException(
'Loading currency base failed. Missing file {}'.format(
e.filename))
def compileFilter(self, fltr, path=None):
if path is None:
path = []
if fltr.id in path:
raise AppException(
"Circular reference detected while compiling filters: {}".
format(path))
path.append(fltr.id)
if not fltr.baseId or fltr.baseId == fltr.id:
baseComp = {}
else:
baseFilter = self.getFilterById(
fltr.baseId, itertools.chain(self.userFilters,
self.autoFilters))
if baseFilter is None:
# try using last compilation
compiledFilter = self.getFilterById(
fltr.baseId, self.activeFilters,
lambda x, y: x.fltr.id == y)
if compiledFilter is None:
raise CompileException(
"Base filter '{}' not found.".format(fltr.baseId))
# return None
baseComp = self.compileFilter(compiledFilter.fltr, path)
else:
baseComp = self.compileFilter(baseFilter, path)
# if baseComp is None:
# return None
comp = fltr.compile(baseComp)
if fltr.id.startswith('_'):
val_override = self.price_overrides.get(
fltr.id, self.default_price_override)
comp['price_max'] = cm.compilePrice(val_override,
comp['price_max'])
# return fltr.compile(baseComp)
return comp
def scan(self):
msgr.send_msg("Scan initializing..")
os.makedirs('tmp', exist_ok=True)
os.makedirs('log', exist_ok=True)
is_beta = config.league.lower().startswith('beta ')
if is_beta:
self.poe_api_url = POE_BETA_API
self.league = re.sub('beta ', '', config.league, flags=re.IGNORECASE)
else:
self.poe_api_url = POE_API
self.league = config.league
# assertions
if not cm.initialized:
raise AppException("Currency information must be initialized before starting a scan.")
if not fm.initialized:
raise AppException("Filters information must be initialized before starting a scan.")
if cm.needUpdate:
try:
cm.update()
msgr.send_msg("Currency rates updated successfully.")
except AppException as e:
msgr.send_msg(e, logging.ERROR)
if cm.initialized:
msgr.send_msg('Using currency information from a local copy..', logging.WARN)
if fm.needUpdate:
try:
msgr.send_msg("Generating filters from API..")
fm.fetchFromAPI()
except AppException as e:
# filterFallback = True
msgr.send_msg(e, logging.ERROR)
msgr.send_msg('Compiling filters..', logging.INFO)
fm.compileFilters(force_validation=True)
filters = fm.getActiveFilters()
if not len(filters):
raise AppException("No filters are active. Stopping..")
self.stateMgr.loadState()
if self.stateMgr.getChangeId() == "" or str(config.scan_mode).lower() == "latest":
msgr.send_msg("Fetching latest id from API..")
latest_id = self._get_latest_id(is_beta)
if latest_id:
if not self.stateMgr.getChangeId() or get_delta(self.stateMgr.getChangeId(), latest_id) > 0:
self.stateMgr.saveState(latest_id)
else:
msgr.send_msg('Saved ID is more recent, continuing..')
elif not self._stop.is_set():
raise AppException("Failed retrieving latest ID from API")
self.updater.start()
self.notifier.start()
get_next = True
msgr.send_msg("Scanning started")
msgr.send_update_id(self.stateMgr.getChangeId())
while not self._stop.is_set():
if self.downloader is None or not self.downloader.is_alive():
if self.downloader:
msgr.send_msg("Download thread ended abruptly. Restarting it..", logging.WARN)
self.downloader = Downloader(self.stateMgr.getChangeId(), conns=config.max_conns)
self.downloader.start()
if self.parser is None or not self.parser.is_alive() and not self.parser.signal_stop:
if self.parser:
msgr.send_msg("Parser thread ended abruptly. Restarting it..", logging.WARN)
if config.num_workers > 0:
workers = config.num_workers
else:
workers = max((os.cpu_count() or 1) - 1, 1)
self.parser = ParserThread(workers, self.league, self.stateMgr, self.handleResult)
self.parser.start()
try:
if get_next:
req_id, resp = self.downloader.get(timeout=0.5)
get_next = False
self.parser.put(req_id, resp, timeout=0.5)
get_next = True
except Full:
msgr.send_msg("Parser queue is full.. waiting for parser..", logging.WARN)
except Empty:
continue
def init(self):
try:
self.load()
except Exception as e:
raise AppException(
'Failed to load item mods information.\n{}'.format(e))
def fetchFromAPI(self, force_update=False, accept_empty=False):
if not force_update and not self.needUpdate:
return
# print('updating filters..')
try:
filter_ids = []
filters = []
def name_to_id(name):
return '_' + name.lower().replace(' ', '_')
def get_unique_id(title, name, category, links):
title_id = name_to_id(title)
if title_id not in filter_ids:
return title_id
name_id = name_to_id('{}{}'.format(
name, ' {}L'.format(links) if links else ''))
if name_id not in filter_ids:
# print('id {} was taken, using name id {} instead'.format(title_id, name_id))
return name_id
category_id = name_to_id(title + ' ' + category)
if category_id not in filter_ids:
# print('id {} was taken, using category id {} instead'.format(title_id, category_id))
return category_id
id = title_id
n = 2
while id in filter_ids:
id = '{}{}'.format(title_id, n)
n += 1
# if n > 2:
# print('id {} was taken, using {} instead'.format(title_id, id))
return id
c = pycurl.Curl()
for url in _URLS:
furl = url.format(config.league)
data = getJsonFromURL(furl, handle=c, max_attempts=3)
if data is None and not accept_empty:
raise AppException(
"Filters update failed. Empty response from server")
if data:
category = re.match(".*Get(.*)Overview",
furl).group(1).lower()
for item in data['lines']:
if item['count'] < self.confidence_level:
continue
priority = FilterPriority.AutoBase
crit = {}
# crit['price_max'] = "{} exalted".format(float(item.get('exaltedValue', 0)))
crit['price_max'] = "{} chaos".format(
float(item.get('chaosValue', 0)))
base = item['baseType'] if category not in (
'essence', ) else None
name = item['name']
if base:
name += ' ' + base
crit['name'] = ['"{}"'.format(name)]
try:
rarity = ItemRarity(item['itemClass'])
crit['rarity'] = [_ITEM_TYPE[rarity]]
except ValueError:
rarity = None
crit['buyout'] = True
if category in ('uniquearmour', 'uniqueweapon'):
crit['corrupted'] = False
links = item['links']
title = "{} {} {}".format(
'Legacy' if rarity == ItemRarity.Relic else '',
item['name'], item['variant']
if item['variant'] is not None else '').strip()
if links:
title = '{} {}L'.format(title, links)
crit['links_min'] = links
if links == 5:
priority += 1
elif links == 6:
priority += 2
tier = item['mapTier']
if tier:
crit['level_min'] = tier
crit['level_max'] = tier
id = get_unique_id(title, name, category, links)
filter_ids.append(id)
fltr = Filter(title,
crit,
False,
category,
id=id,
priority=priority)
if item['variant'] is not None:
if item['variant'] not in _VARIANTS:
msgr.send_msg(
"Unknown variant {} in item {}".format(
item['variant'], item['name']),
logging.WARN)
else:
# crit['explicit'] = {'mods': [{'expr': _VARIANTS[item['variant']]}]}
mfs = _VARIANTS[item['variant']]
if mfs:
fg = AllFilterGroup()
for expr in _VARIANTS[item['variant']]:
fg.addModFilter(
ModFilter(ModFilterType.Explicit,
expr))
fltr.criteria['fgs'] = [fg.toDict()]
fltr.validate()
filters.append(fltr)
self.autoFilters = filters
self.item_prices = self.getPrices(self.autoFilters)
self.saveAutoFilters()
self.last_update = datetime.utcnow() if filters else None
except pycurl.error as e:
raise AppException(
"Filters update failed. Connection error: {}".format(e))
except (KeyError, ValueError) as e:
raise AppException(
"Filters update failed. Parsing error: {}".format(e))
except AppException:
raise
except Exception as e:
logexception()
raise AppException(
"Filters update failed. Unexpected error: {}".format(e))