def load_overrides(path, custom=False):
""" Overrides loader for Python files
Note:
Overrides must be in an ``overrides`` dictionary.
Args:
path (str): Path to Python file to be loaded
custom (bool): Boolean flag to specify if this is a custom overrides file
"""
try:
if custom:
sys.path.append(path)
from overrides import overrides
else:
from burst_overrides import overrides
if custom:
log.debug("Imported overrides: %s", repr(overrides))
for provider in overrides:
update_definitions(provider, overrides[provider])
if custom:
log.info("Successfully loaded overrides from %s",
os.path.join(path, "overrides.py"))
except Exception as e:
import traceback
log.error("Failed importing %soverrides: %s",
"custom " if custom else "", repr(e))
map(log.error, traceback.format_exc().split("\n"))
def parse_payload(method, payload):
if method == 'general':
if 'query' in payload:
payload['title'] = payload['query']
payload['titles'] = {'source': payload['query']}
else:
payload = {
'title': payload,
'titles': {
'source': payload
},
}
payload['titles'] = dict(
(k.lower(), v) for k, v in payload['titles'].iteritems())
if get_setting('kodi_language', bool):
kodi_language = xbmc.getLanguage(xbmc.ISO_639_1)
if not kodi_language:
log.warning("Kodi returned empty language code...")
elif kodi_language not in payload.get('titles', {}):
log.info("No '%s' translation available..." % kodi_language)
else:
payload["search_title"] = payload["titles"][kodi_language]
if "search_title" not in payload:
log.info(
"Could not determine search title, falling back to normal title: %s",
repr(payload["title"]))
payload["search_title"] = payload["title"]
return payload
def get_cloudhole_clearance(cloudhole_key):
""" CloudHole clearance fetcher
Args:
cloudhole_key (str): The CloudHole API key saved in settings or from ``get_cloudhole_key`` directly
Returns:
tuple: A CloudHole clearance cookie and user-agent string
"""
user_agent = USER_AGENT
clearance = None
if cloudhole_key:
try:
r = urllib2.Request("https://cloudhole.herokuapp.com/clearances")
r.add_header('Content-type', 'application/json')
r.add_header('Authorization', cloudhole_key)
with closing(urllib2.urlopen(r)) as response:
content = response.read()
log.debug("CloudHole returned: %s" % content)
data = json.loads(content)
user_agent = data[0]['userAgent']
clearance = data[0]['cookies']
log.info("New UA and clearance: %s / %s" % (user_agent, clearance))
except Exception as e:
log.error("CloudHole error: %s" % repr(e))
return clearance, user_agent
def got_results(provider, results):
""" Results callback once a provider found all its results, or not
Args:
provider (str): The provider ID
results (list): The list of results
"""
global provider_names
global provider_results
global available_providers
definition = definitions[provider]
definition = get_alias(definition, get_setting("%s_alias" % provider))
max_results = get_setting('max_results', int)
sorted_results = sorted(results, key=lambda r: (r['seeds']), reverse=True)
if len(sorted_results) > max_results:
sorted_results = sorted_results[:max_results]
log.info(">> %s returned %2d results in %.1f seconds%s" %
(definition['name'].rjust(longest), len(results),
round(time.time() - request_time, 2),
(", sending %d best ones" %
max_results) if len(results) > max_results else ""))
provider_results.extend(sorted_results)
available_providers -= 1
if definition['name'] in provider_names:
provider_names.remove(definition['name'])
def search_jackett(payload, method):
jackett = get_client()
if jackett is None:
utils.notify(utils.translation(32603), image=utils.get_icon_path())
return []
log.debug("Processing %s with Jackett", method)
if method == 'movie':
res = jackett.search_movie(payload["search_title"], payload['year'],
payload["imdb_id"])
elif method == 'season':
res = jackett.search_season(payload["search_title"], payload["season"],
payload["imdb_id"])
elif method == 'episode':
res = jackett.search_episode(payload["search_title"],
payload["season"], payload["episode"],
payload["imdb_id"])
elif method == 'anime':
log.warning("jackett provider does not yet support anime search")
res = []
log.info("anime payload=%s", repr(payload))
# client.search_query(payload["search_title"], payload["season"], payload["episode"], payload["imdb_id"])
else:
res = jackett.search_query(payload["search_title"])
log.debug("%s search returned %d results", method, len(res))
res = filter_results(method, res)
return sort_results(res)
def resolution(results):
filtered = []
for result in results:
log.info("res %s: name=%s; id=%d", result['name'],
result['_resolution'], result['resolution'])
if get_setting('include_resolution_' + result["_resolution"], bool):
filtered.append(result)
return filtered
def load():
from elementum.provider import log
from utils import get_setting
if get_setting("enable_debugger", bool):
import pkgutil
import re
from os import path
import sys
additional_libraries = get_setting("debugger_additional_libraries")
if additional_libraries != "":
if not path.exists(additional_libraries):
log.error(
"Debugger has been enabled but additional libraries directory, skipping loading of debugger"
)
return
sys.path.append(additional_libraries)
if pkgutil.find_loader("pydevd_pycharm") is None:
log.error(
"Debugger currently only supports IntelliJ IDEA and derivatives. If you need additional "
)
return
host = get_setting("debugger_host")
valid_host_regex = re.compile(
r'''
^
(?:
(?:(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))
|
(?:(?:(?:[a-zA-Z]|[a-zA-Z][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)+(?:[A-Za-z|[A-Za-z][A-Za-z0-9\-]*[A-Za-z0-9]))
)
$
''', re.VERBOSE)
if not valid_host_regex.match(host):
log.error("debugger: invalid host detected.. Skipping")
return False
try:
port = get_setting("debugger_port", int)
except ValueError:
log.exception("debugger: invalid port detected")
return
if not (0 < int(port) <= 65535):
log.exception("debugger: port must be between 0 and 65535")
return
import pydevd_pycharm
pydevd_pycharm.settrace(host,
port=port,
stdoutToServer=True,
stderrToServer=True)
log.info("pycharm debugger successfully loaded")
def _parse_items(self, resp_content):
results = []
xml = ET.ElementTree(ET.fromstring(resp_content))
items = xml.getroot().findall("channel/item")
log.info("Found %d items from response", len(items))
for item in items:
result = self._parse_item(item)
if result is not None:
results.append(result)
return results
def got_results(provider, results):
""" Results callback once a provider found all its results, or not
Args:
provider (str): The provider ID
results (list): The list of results
"""
global provider_names
global provider_results
global available_providers
definition = definitions[provider]
definition = get_alias(definition, get_setting("%s_alias" % provider))
max_results = get_setting('max_results', int)
sort_by = get_setting('sort_by', int)
# 0 "Resolution"
# 1 "Seeds"
# 2 "Size"
# 3 "Balanced"
if not sort_by or sort_by == 3 or sort_by > 3:
# TODO: think of something interesting to balance sort results
sorted_results = sorted(results,
key=lambda r: (nonesorter(r['sort_balance'])),
reverse=True)
elif sort_by == 0:
sorted_results = sorted(results,
key=lambda r:
(nonesorter(r['sort_resolution'])),
reverse=True)
elif sort_by == 1:
sorted_results = sorted(results,
key=lambda r: (nonesorter(r['seeds'])),
reverse=True)
elif sort_by == 2:
sorted_results = sorted(results,
key=lambda r: (nonesorter(r['size'])),
reverse=True)
if len(sorted_results) > max_results:
sorted_results = sorted_results[:max_results]
log.info("[%s] >> %s returned %2d results in %.1f seconds%s" %
(provider, definition['name'].rjust(longest), len(results),
round(time.time() - request_time, 2),
(", sending %d best ones" %
max_results) if len(results) > max_results else ""))
provider_results.extend(sorted_results)
available_providers -= 1
if definition['name'] in provider_names:
provider_names.remove(definition['name'])
def get_cloudhole_key():
""" CloudHole API key fetcher
Returns:
str: A CloudHole API key
"""
cloudhole_key = None
try:
r = urllib2.Request("https://cloudhole.herokuapp.com/key")
r.add_header('Content-type', 'application/json')
with closing(urllib2.urlopen(r)) as response:
content = response.read()
log.info("CloudHole key: %s" % content)
data = json.loads(content)
cloudhole_key = data['key']
except Exception as e:
log.error("Getting CloudHole key error: %s" % repr(e))
return cloudhole_key
def got_results(provider, results):
""" Results callback once a provider found all its results, or not
Args:
provider (str): The provider ID
results (list): The list of results
"""
global provider_names
global provider_results
global available_providers
definition = definitions[provider]
definition = get_alias(definition, get_setting("%s_alias" % provider))
max_results = get_setting('max_results', int)
if get_setting('sort_by_resolution', bool):
log.debug(
"[%s][EXPEREMENTAL] Sorting by resolution before cutoff max_results"
% provider)
sorted_results = sorted(results,
key=lambda r: (r['resolution']),
reverse=True)
else:
sorted_results = sorted(results,
key=lambda r: (r['seeds']),
reverse=True)
if get_setting('disable_max', bool):
log.debug('[%s] Don\'t apply "max_results" settings' % provider)
max_results = 999
elif len(sorted_results) > max_results:
sorted_results = sorted_results[:max_results]
log.info(">> %s returned %2d results in %.1f seconds%s" %
(definition['name'].rjust(longest), len(results),
round(time.time() - request_time, 2),
(", sending %d best ones" %
max_results) if len(results) > max_results else ""))
provider_results.extend(sorted_results)
available_providers -= 1
if definition['name'] in provider_names:
provider_names.remove(definition['name'])
def get_caps(self):
caps_resp = self._session.get("all/results/torznab",
params={
"t": "caps",
"apikey": self._api_key
})
if caps_resp.status_code != httplib.OK:
notify(translation(32700).format(caps_resp.reason),
image=get_icon_path())
log.error("Jackett return %s", caps_resp.reason)
set_setting('settings_validated', caps_resp.reason)
return
err = self.get_error(caps_resp.content)
if err is not None:
notify(translation(32700).format(err["description"]),
image=get_icon_path())
log.error("got code %s: %s", err["code"], err["description"])
set_setting('settings_validated', err["description"])
return
set_setting('settings_validated', 'Success')
xml = ET.ElementTree(ET.fromstring(caps_resp.content)).getroot()
# todo handle gracefully, doesn't exist for individual trackers
# self._caps["limits"] = xml.find("limits").attrib
self._caps["search_tags"] = {}
for type_tag in xml.findall('searching/*'):
self._caps["search_tags"][type_tag.tag] = {
"enabled":
type_tag.attrib["available"] == "yes",
"params": [
p for p in type_tag.attrib['supportedParams'].split(",")
if p
],
}
log.info("Found capabilities: %s", repr(self._caps))
def search(payload, method="general"):
payload = parse_payload(method, payload)
log.debug("Searching with payload (%s): %s", method, repr(payload))
p_dialog = xbmcgui.DialogProgressBG()
p_dialog.create('Elementum [COLOR FFFF6B00]Jackett[/COLOR]',
utils.translation(32602))
try:
request_start_time = time.time()
results = search_jackett(payload, method)
request_end_time = time.time()
request_time = round(request_end_time - request_start_time, 2)
log.debug("All results: %s" % repr(results))
log.info("Jackett returned %d results in %s seconds", len(results),
request_time)
finally:
p_dialog.close()
del p_dialog
return results
def process_keywords(self, provider, text):
""" Processes the query payload from a provider's keyword definitions
Args:
provider (str): Provider ID
text (str): Keyword placeholders from definitions, ie. {title}
Returns:
str: Processed query keywords
"""
keywords = self.read_keywords(text)
replacing = get_setting("filter_quotes", bool)
for keyword in keywords:
keyword = keyword.lower()
if 'title' in keyword:
title = self.info["title"]
language = definitions[provider]['language']
use_language = None
if ':' in keyword:
use_language = keyword.split(':')[1].lower()
if provider not in self.language_exceptions and \
(use_language or self.kodi_language) and \
'titles' in self.info and self.info['titles']:
try:
if self.kodi_language and self.kodi_language in self.info[
'titles']:
use_language = self.kodi_language
if use_language not in self.info['titles']:
use_language = language
if 'original' in self.info['titles']:
title = self.info['titles']['original']
if use_language in self.info['titles'] and self.info[
'titles'][use_language]:
title = self.info['titles'][use_language]
title = normalize_string(title)
log.info("[%s] Using translated '%s' title %s" %
(provider, use_language, repr(title)))
log.debug(
"[%s] Translated titles from Elementum: %s" %
(provider, repr(self.info['titles'])))
except Exception as e:
import traceback
log.error("%s failed with: %s" % (provider, repr(e)))
map(log.debug, traceback.format_exc().split("\n"))
text = text.replace('{%s}' % keyword, title)
if 'year' in keyword:
text = text.replace('{%s}' % keyword, str(self.info["year"]))
if 'season' in keyword:
if '+' in keyword:
keys = keyword.split('+')
season = str(self.info["season"] + get_int(keys[1]))
elif ':' in keyword:
keys = keyword.split(':')
season = ('%%.%sd' % keys[1]) % self.info["season"]
else:
season = '%s' % self.info["season"]
text = text.replace('{%s}' % keyword, season)
if 'episode' in keyword:
if '+' in keyword:
keys = keyword.split('+')
episode = str(self.info["episode"] + get_int(keys[1]))
elif ':' in keyword:
keys = keyword.split(':')
episode = ('%%.%sd' % keys[1]) % self.info["episode"]
else:
episode = '%s' % self.info["episode"]
text = text.replace('{%s}' % keyword, episode)
if replacing:
text = text.replace(u"'", '')
return text
def search(payload, method="general"):
""" Main search entrypoint
Args:
payload (dict): Search payload from Elementum.
method (str): Type of search, can be ``general``, ``movie``, ``show``, ``season`` or ``anime``
Returns:
list: All filtered results in the format Elementum expects
"""
log.debug("Searching with payload (%s): %s" % (method, repr(payload)))
if method == 'general':
if 'query' in payload:
payload['title'] = payload['query']
payload['titles'] = {'source': payload['query']}
else:
payload = {
'title': payload,
'titles': {
'source': payload
},
}
payload['titles'] = dict(
(k.lower(), v) for k, v in payload['titles'].iteritems())
# If titles[] exists in payload and there are special chars in titles[source]
# then we set a flag to possibly modify the search query
payload['has_special'] = 'titles' in payload and \
bool(payload['titles']) and \
'source' in payload['titles'] and \
any(c in payload['titles']['source'] for c in special_chars)
if payload['has_special']:
log.debug(
"Query title contains special chars, so removing any quotes in the search query"
)
if 'proxy_url' not in payload:
payload['proxy_url'] = ''
if 'internal_proxy_url' not in payload:
payload['internal_proxy_url'] = ''
if 'elementum_url' not in payload:
payload['elementum_url'] = ''
if 'silent' not in payload:
payload['silent'] = False
if 'skip_auth' not in payload:
payload['skip_auth'] = False
global request_time
global provider_names
global provider_results
global available_providers
provider_names = []
provider_results = []
available_providers = 0
request_time = time.time()
providers = get_enabled_providers(method)
if len(providers) == 0:
if not payload['silent']:
notify(translation(32060), image=get_icon_path())
log.error("No providers enabled")
return []
log.info(
"Burstin' with %s" %
", ".join([definitions[provider]['name'] for provider in providers]))
if get_setting('kodi_language', bool):
kodi_language = xbmc.getLanguage(xbmc.ISO_639_1)
if not kodi_language:
log.warning("Kodi returned empty language code...")
elif 'titles' not in payload or not payload['titles']:
log.info("No translations available...")
elif payload['titles'] and kodi_language not in payload['titles']:
log.info("No '%s' translation available..." % kodi_language)
p_dialog = xbmcgui.DialogProgressBG()
if not payload['silent']:
p_dialog.create('Elementum [COLOR FFFF6B00]Burst[/COLOR]',
translation(32061))
for provider in providers:
available_providers += 1
provider_names.append(definitions[provider]['name'])
task = Thread(target=run_provider, args=(provider, payload, method))
task.start()
providers_time = time.time()
total = float(available_providers)
# Exit if all providers have returned results or timeout reached, check every 100ms
while time.time() - providers_time < timeout and available_providers > 0:
timer = time.time() - providers_time
log.debug("Timer: %ds / %ds" % (timer, timeout))
if timer > timeout:
break
message = translation(
32062
) % available_providers if available_providers > 1 else translation(
32063)
if not payload['silent']:
p_dialog.update(int((total - available_providers) / total * 100),
message=message)
time.sleep(0.25)
if not payload['silent']:
p_dialog.close()
del p_dialog
if available_providers > 0:
message = u', '.join(provider_names)
message = message + translation(32064)
log.warning(message.encode('utf-8'))
if not payload['silent']:
notify(message, ADDON_ICON)
log.debug("all provider_results: %s" % repr(provider_results))
filtered_results = apply_filters(provider_results)
log.debug("all filtered_results: %s" % repr(filtered_results))
log.info("Providers returned %d results in %s seconds" %
(len(filtered_results), round(time.time() - request_time, 2)))
return filtered_results
load_providers(os.path.join(ADDON_PATH, 'burst', 'providers',
'providers.json'))
# Load providers overrides
load_overrides(os.path.join(ADDON_PATH, 'burst', 'providers'))
# Load user's custom providers
custom_providers = os.path.join(xbmc.translatePath(ADDON_PROFILE), "providers")
if not os.path.exists(custom_providers):
try:
os.makedirs(custom_providers)
except Exception as e:
log.error("Unable to create custom providers folder: %s", repr(e))
pass
for provider_file in glob(os.path.join(custom_providers, "*.json")):
log.info("Importing and enabling %s" % provider_file)
load_providers(provider_file, custom=True)
# Load user's custom overrides
custom_overrides = xbmc.translatePath(ADDON_PROFILE)
if os.path.exists(os.path.join(custom_overrides, 'overrides.py')):
load_overrides(custom_overrides, custom=True)
# Load json overrides
load_providers(
os.path.join(xbmc.translatePath(ADDON_PROFILE), 'overrides.json'))
# Setting mandatory fields to their default values for each provider.
for provider in definitions:
for k, v in iteritems(mandatory_fields):
if k not in definitions[provider]:
def process_keywords(self, provider, text, definition):
""" Processes the query payload from a provider's keyword definitions
Args:
provider (str): Provider ID
text (str): Keyword placeholders from definitions, ie. {title}
Returns:
str: Processed query keywords
"""
keywords = self.read_keywords(text)
replacing = use_filter_quotes
for keyword in keywords:
keyword = keyword.lower()
if 'title' in keyword:
title = self.info["title"]
language = definitions[provider]['language']
use_language = None
if ':' in keyword:
use_language = keyword.split(':')[1].lower()
if provider not in self.language_exceptions and \
(use_language or self.kodi_language) and \
'titles' in self.info and self.info['titles']:
try:
if not use_language and self.kodi_language and self.kodi_language in self.info[
'titles']:
use_language = self.kodi_language
if not use_language and language and language in self.info[
'titles']:
use_language = language
if use_language in self.info['titles'] and self.info[
'titles'][use_language]:
title = self.info['titles'][use_language]
title = normalize_string(title)
# For all non-original titles, try to remove accents from the title.
if use_language != 'original':
title = remove_accents(title)
# Remove characters, filled in 'remove_special_characters' field definition.
if 'remove_special_characters' in definition and definition[
'remove_special_characters']:
for char in definition[
'remove_special_characters']:
title = title.replace(char, "")
title = " ".join(title.split())
log.info("[%s] Using translated '%s' title %s" %
(provider, use_language, repr(title)))
else:
log.debug(
"[%s] Skipping the query '%s' due to missing '%s' language title"
% (provider, text, use_language))
# If title for specific language cannot be read - cancel this query
return ""
except Exception as e:
import traceback
log.error("%s failed with: %s" % (provider, repr(e)))
map(log.debug, traceback.format_exc().split("\n"))
text = text.replace('{%s}' % keyword, title)
if 'year' in keyword:
text = text.replace('{%s}' % keyword, str(self.info["year"]))
if 'show_tmdb_id' in keyword:
if 'show_tmdb_id' not in self.info:
self.info['show_tmdb_id'] = ''
text = text.replace('{%s}' % keyword,
str(self.info["show_tmdb_id"]))
if 'tmdb_id' in keyword:
if 'tmdb_id' not in self.info:
self.info['tmdb_id'] = ''
text = text.replace('{%s}' % keyword,
str(self.info["tmdb_id"]))
if 'tvdb_id' in keyword:
if 'tvdb_id' not in self.info:
self.info['tvdb_id'] = ''
text = text.replace('{%s}' % keyword,
str(self.info["tvdb_id"]))
if 'imdb_id' in keyword:
if 'imdb_id' not in self.info:
self.info['imdb_id'] = ''
text = text.replace('{%s}' % keyword,
str(self.info["imdb_id"]))
if 'season' in keyword:
if '+' in keyword:
keys = keyword.split('+')
season = str(self.info["season"] + get_int(keys[1]))
elif ':' in keyword:
keys = keyword.split(':')
season = ('%%.%sd' % keys[1]) % self.info["season"]
else:
season = '%s' % self.info["season"]
text = text.replace('{%s}' % keyword, season)
if 'episode' in keyword:
if '+' in keyword:
keys = keyword.split('+')
episode = str(self.info["episode"] + get_int(keys[1]))
elif ':' in keyword:
keys = keyword.split(':')
episode = ('%%.%sd' % keys[1]) % self.info["episode"]
else:
episode = '%s' % self.info["episode"]
text = text.replace('{%s}' % keyword, episode)
if replacing:
text = text.replace(u"'", '')
return text
auto_timeout = get_setting("auto_timeout", bool)
timeout = get_setting("timeout", int)
debug_parser = get_setting("use_debug_parser", bool)
max_results = get_setting('max_results', int)
sort_by = get_setting('sort_by', int)
special_chars = "()\"':.[]<>/\\?"
elementum_timeout = 0
elementum_addon = xbmcaddon.Addon(id='plugin.video.elementum')
if elementum_addon:
if elementum_addon.getSetting('custom_provider_timeout_enabled') == "true":
elementum_timeout = int(elementum_addon.getSetting('custom_provider_timeout'))
else:
elementum_timeout = 30
log.info("Using timeout from Elementum: %d seconds" % (elementum_timeout))
# Make sure timeout is always less than the one from Elementum.
if auto_timeout:
timeout = elementum_timeout - 3
elif elementum_timeout > 0 and timeout > elementum_timeout - 3:
log.info("Redefining timeout to be less than Elementum's: %d to %d seconds" % (timeout, elementum_timeout - 3))
timeout = elementum_timeout - 3
def search(payload, method="general"):
""" Main search entrypoint
Args:
payload (dict): Search payload from Elementum.
method (str): Type of search, can be ``general``, ``movie``, ``show``, ``season`` or ``anime``
def process(provider,
generator,
filtering,
has_special,
verify_name=True,
verify_size=True):
""" Method for processing provider results using its generator and Filtering class instance
Args:
provider (str): Provider ID
generator (function): Generator method, can be either ``extract_torrents`` or ``extract_from_api``
filtering (Filtering): Filtering class instance
has_special (bool): Whether title contains special chars
verify_name (bool): Whether to double-check the results' names match the query or not
verify_size (bool): Whether to check the results' file sizes
"""
log.debug("execute_process for %s with %s" % (provider, repr(generator)))
definition = definitions[provider]
definition = get_alias(definition, get_setting("%s_alias" % provider))
client = Client(info=filtering.info)
logged_in = False
if get_setting('kodi_language', bool):
kodi_language = xbmc.getLanguage(xbmc.ISO_639_1)
if kodi_language:
filtering.kodi_language = kodi_language
language_exceptions = get_setting('language_exceptions')
if language_exceptions.strip().lower():
filtering.language_exceptions = re.split(r',\s?',
language_exceptions)
log.debug("[%s] Queries: %s" % (provider, filtering.queries))
log.debug("[%s] Extras: %s" % (provider, filtering.extras))
for query, extra in zip(filtering.queries, filtering.extras):
log.debug("[%s] Before keywords - Query: %s - Extra: %s" %
(provider, repr(query), repr(extra)))
if has_special:
# Removing quotes, surrounding {title*} keywords, when title contains special chars
query = re.sub("[\"']({title.*?})[\"']", '\\1', query)
query = filtering.process_keywords(provider, query)
extra = filtering.process_keywords(provider, extra)
if 'charset' in definition and 'utf' not in definition[
'charset'].lower():
try:
query = urllib.quote(query.encode(definition['charset']))
extra = urllib.quote(extra.encode(definition['charset']))
except:
pass
log.debug("[%s] After keywords - Query: %s - Extra: %s" %
(provider, repr(query), repr(extra)))
if not query:
return filtering.results
query = urllib.quote(query.encode('utf-8'))
url_search = filtering.url.replace('QUERY', query)
if extra:
url_search = url_search.replace('EXTRA', extra)
else:
url_search = url_search.replace('EXTRA', '')
url_search = url_search.replace(' ', definition['separator'])
if 'post_data' in definition and not filtering.post_data:
filtering.post_data = eval(definition['post_data'])
# Creating the payload for POST method
payload = dict()
for key, value in filtering.post_data.iteritems():
if 'QUERY' in value:
payload[key] = filtering.post_data[key].replace('QUERY', query)
else:
payload[key] = filtering.post_data[key]
# Creating the payload for GET method
data = None
if filtering.get_data:
data = dict()
for key, value in filtering.get_data.iteritems():
if 'QUERY' in value:
data[key] = filtering.get_data[key].replace('QUERY', query)
else:
data[key] = filtering.get_data[key]
log.debug("- %s query: %s" % (provider, repr(query)))
log.debug("-- %s url_search before token: %s" %
(provider, repr(url_search)))
log.debug("--- %s using POST payload: %s" % (provider, repr(payload)))
log.debug("----%s filtering with post_data: %s" %
(provider, repr(filtering.post_data)))
# Set search's "title" in filtering to double-check results' names
if 'filter_title' in definition and definition['filter_title']:
filtering.filter_title = True
filtering.title = query
if logged_in:
log.info("[%s] Reusing previous login" % provider)
elif 'private' in definition and definition['private']:
username = get_setting('%s_username' % provider)
password = get_setting('%s_password' % provider)
if 'login_object' in definition and definition['login_object']:
logged_in = False
try:
login_object = definition['login_object'].replace(
'USERNAME',
'"%s"' % username).replace('PASSWORD',
'"%s"' % password)
except Exception, e:
log.error("[{0}] Make login_object fail: {1}".format(
provider, e))
return filtering.results
# TODO generic flags in definitions for those...
if provider == 'lostfilm':
client.open(definition['root_url'] +
'/v_search.php?c=110&s=1&e=1')
if u'Вход. – LostFilm.TV.' in client.content:
pass
else:
log.info('[%s] Login successful' % provider)
logged_in = True
if not logged_in and client.login(
definition['root_url'] + definition['login_path'],
eval(login_object), definition['login_failed']):
log.info('[%s] Login successful' % provider)
logged_in = True
elif not logged_in:
log.error("[%s] Login failed: %s", provider, client.status)
log.debug("[%s] Failed login content: %s", provider,
repr(client.content))
notify(translation(32089).format(provider),
image=get_icon_path())
return filtering.results
if logged_in:
if provider == 'lostfilm':
log.info('[%s] Search lostfilm serial ID...', provider)
url_search = fix_lf(url_search)
client.open(url_search.encode('utf-8'),
post_data=payload,
get_data=data)
series_details = re.search(
r'"mark-rate-pane" rel="(\d+),(\d+),(\d+)">',
client.content)
if series_details:
client.open(definition['root_url'] +
'/v_search.php?a=%s%s%s' %
(series_details.group(1),
series_details.group(2).zfill(3),
series_details.group(3).zfill(3)))
redirect_url = re.search(ur'url=(.*?)">',
client.content)
if redirect_url is not None:
url_search = redirect_url.group(1)
else:
log.info('[%s] Not found ID in %s' %
(provider, url_search))
return filtering.results
log.info("> %s search URL: %s" %
(definition['name'].rjust(longest), url_search))
client.open(url_search, post_data=payload, get_data=data)
filtering.results.extend(
generate_payload(provider, generator(provider, client), filtering,
verify_name, verify_size))
def process(provider,
generator,
filtering,
has_special,
verify_name=True,
verify_size=True):
""" Method for processing provider results using its generator and Filtering class instance
Args:
provider (str): Provider ID
generator (function): Generator method, can be either ``extract_torrents`` or ``extract_from_api``
filtering (Filtering): Filtering class instance
has_special (bool): Whether title contains special chars
verify_name (bool): Whether to double-check the results' names match the query or not
verify_size (bool): Whether to check the results' file sizes
"""
log.debug("execute_process for %s with %s" % (provider, repr(generator)))
definition = definitions[provider]
definition = get_alias(definition, get_setting("%s_alias" % provider))
client = Client()
token = None
logged_in = False
token_auth = False
if get_setting("use_cloudhole", bool):
client.clearance = get_setting('clearance')
client.user_agent = get_setting('user_agent')
if get_setting('kodi_language', bool):
kodi_language = xbmc.getLanguage(xbmc.ISO_639_1)
if kodi_language:
filtering.kodi_language = kodi_language
language_exceptions = get_setting('language_exceptions')
if language_exceptions.strip().lower():
filtering.language_exceptions = re.split(r',\s?',
language_exceptions)
log.debug("[%s] Queries: %s" % (provider, filtering.queries))
log.debug("[%s] Extras: %s" % (provider, filtering.extras))
for query, extra in zip(filtering.queries, filtering.extras):
log.debug("[%s] Before keywords - Query: %s - Extra: %s" %
(provider, repr(query), repr(extra)))
if has_special:
# Removing quotes, surrounding {title*} keywords, when title contains special chars
query = re.sub("[\"']({title.*?})[\"']", '\\1', query)
query = filtering.process_keywords(provider, query)
extra = filtering.process_keywords(provider, extra)
if 'charset' in definition and 'utf' not in definition[
'charset'].lower():
try:
query = urllib.quote(query.encode(definition['charset']))
extra = urllib.quote(extra.encode(definition['charset']))
except:
pass
log.debug("[%s] After keywords - Query: %s - Extra: %s" %
(provider, repr(query), repr(extra)))
if not query:
return filtering.results
url_search = filtering.url.replace('QUERY', query)
if extra:
url_search = url_search.replace('EXTRA', extra)
else:
url_search = url_search.replace('EXTRA', '')
url_search = url_search.replace(' ', definition['separator'])
# MagnetDL fix...
url_search = url_search.replace('FIRSTLETTER', query[:1])
# Creating the payload for POST method
if 'post_data' in definition and not filtering.post_data:
filtering.post_data = eval(definition['post_data'])
payload = dict()
for key, value in filtering.post_data.iteritems():
if 'QUERY' in value:
payload[key] = filtering.post_data[key].replace('QUERY', query)
else:
payload[key] = filtering.post_data[key]
# Creating the payload for GET method
data = None
if filtering.get_data:
data = dict()
for key, value in filtering.get_data.iteritems():
if 'QUERY' in value:
data[key] = filtering.get_data[key].replace('QUERY', query)
else:
data[key] = filtering.get_data[key]
log.debug("- %s query: %s" % (provider, repr(query)))
log.debug("-- %s url_search before token: %s" %
(provider, repr(url_search)))
log.debug("--- %s using POST payload: %s" % (provider, repr(payload)))
log.debug("----%s filtering with post_data: %s" %
(provider, repr(filtering.post_data)))
# Set search's "title" in filtering to double-check results' names
if 'filter_title' in definition and definition['filter_title']:
filtering.filter_title = True
filtering.title = query
if token:
log.info('[%s] Reusing existing token' % provider)
url_search = url_search.replace('TOKEN', token)
elif 'token' in definition:
token_url = definition['base_url'] + definition['token']
log.debug("Getting token for %s at %s" %
(provider, repr(token_url)))
client.open(token_url.encode('utf-8'))
try:
token_data = json.loads(client.content)
except:
log.error('%s: Failed to get token for %s' %
(provider, repr(url_search)))
return filtering.results
log.debug("Token response for %s: %s" %
(provider, repr(token_data)))
if 'token' in token_data:
token = token_data['token']
log.debug("Got token for %s: %s" % (provider, repr(token)))
url_search = url_search.replace('TOKEN', token)
else:
log.warning('%s: Unable to get token for %s' %
(provider, repr(url_search)))
if logged_in:
log.info("[%s] Reusing previous login" % provider)
elif token_auth:
log.info("[%s] Reusing previous token authorization" % provider)
elif 'private' in definition and definition['private']:
username = get_setting('%s_username' % provider)
password = get_setting('%s_password' % provider)
passkey = get_setting('%s_passkey' % provider)
if not username and not password and not passkey:
for addon_name in ('script.magnetic.%s' % provider,
'script.magnetic.%s-mc' % provider):
for setting in ('username', 'password'):
try:
value = xbmcaddon.Addon(addon_name).getSetting(
setting)
set_setting('%s_%s' % (provider, setting), value)
if setting == 'username':
username = value
if setting == 'password':
password = value
except:
pass
if passkey:
logged_in = True
client.passkey = passkey
url_search = url_search.replace('PASSKEY', passkey)
elif 'login_object' in definition and definition['login_object']:
logged_in = False
login_object = definition['login_object'].replace(
'USERNAME',
'"%s"' % username).replace('PASSWORD', '"%s"' % password)
# TODO generic flags in definitions for those...
if provider == 'hd-torrents':
client.open(definition['root_url'] +
definition['login_path'])
if client.content:
csrf_token = re.search(
r'name="csrfToken" value="(.*?)"', client.content)
if csrf_token:
login_object = login_object.replace(
'CSRF_TOKEN', '"%s"' % csrf_token.group(1))
else:
logged_in = True
if provider == 'lostfilm':
client.open(definition['root_url'] +
'/v_search.php?c=111&s=1&e=1')
if client.content is not 'log in first':
logged_in = True
if 'token_auth' in definition:
# log.debug("[%s] logging in with: %s" % (provider, login_object))
if client.open(definition['root_url'] +
definition['token_auth'],
post_data=eval(login_object)):
try:
token_data = json.loads(client.content)
except:
log.error('%s: Failed to get token from %s' %
(provider, definition['token_auth']))
return filtering.results
log.debug("Token response for %s: %s" %
(provider, repr(token_data)))
if 'token' in token_data:
client.token = token_data['token']
log.debug("Auth token for %s: %s" %
(provider, repr(client.token)))
else:
log.error('%s: Unable to get auth token for %s' %
(provider, repr(url_search)))
return filtering.results
log.info('[%s] Token auth successful' % provider)
token_auth = True
else:
log.error("[%s] Token auth failed with response: %s" %
(provider, repr(client.content)))
return filtering.results
elif not logged_in and client.login(
definition['root_url'] + definition['login_path'],
eval(login_object), definition['login_failed']):
log.info('[%s] Login successful' % provider)
logged_in = True
elif not logged_in:
log.error("[%s] Login failed: %s", provider, client.status)
log.debug("[%s] Failed login content: %s", provider,
repr(client.content))
return filtering.results
if logged_in:
if provider == 'hd-torrents':
client.open(definition['root_url'] + '/torrents.php')
csrf_token = re.search(
r'name="csrfToken" value="(.*?)"', client.content)
url_search = url_search.replace(
"CSRF_TOKEN", csrf_token.group(1))
if provider == 'lostfilm':
log.info('[%s] Need open page before search', provider)
client.open(url_search.encode('utf-8'),
post_data=payload,
get_data=data)
search_info = re.search(r'PlayEpisode\((.*?)\)">',
client.content)
if search_info:
series_details = re.search(
'\'(\d+)\',\'(\d+)\',\'(\d+)\'',
search_info.group(1))
client.open(definition['root_url'] +
'/v_search.php?c=%s&s=%s&e=%s' %
(series_details.group(1),
series_details.group(2),
series_details.group(3)))
redirect_url = re.search(ur'url=(.*?)">',
client.content)
if redirect_url is not None:
url_search = redirect_url.group(1)
else:
return filtering.results
log.info("> %s search URL: %s" %
(definition['name'].rjust(longest), url_search))
client.open(url_search.encode('utf-8'),
post_data=payload,
get_data=data)
filtering.results.extend(
generate_payload(provider, generator(provider, client), filtering,
verify_name, verify_size))
return filtering.results
def open(self,
url,
language='en',
post_data=None,
get_data=None,
headers=None,
proxy_url=None,
charset='utf8'):
""" Opens a connection to a webpage and saves its HTML content in ``self.content``
Args:
url (str): The URL to open
language (str): The language code for the ``Content-Language`` header
post_data (dict): POST data for the request
get_data (dict): GET data for the request
"""
if not post_data:
post_data = {}
if get_data:
url += '?' + urlencode(get_data)
log.debug("Opening URL: %s" % repr(url))
result = False
data = urlencode(post_data) if len(post_data) > 0 else None
req = urllib2.Request(url, data)
self._read_cookies(url)
log.debug("Cookies for %s: %s" % (repr(url), repr(self._cookies)))
# Parsing proxy information
proxy = {
'enabled': get_setting("proxy_enabled", bool),
'use_type': get_setting("proxy_use_type", int),
'type': proxy_types[0],
'host': get_setting("proxy_host", unicode),
'port': get_setting("proxy_port", int),
'login': get_setting("proxy_login", unicode),
'password': get_setting("proxy_password", unicode),
}
try:
proxy['type'] = proxy_types[get_setting("proxy_type", int)]
except:
pass
handlers = [urllib2.HTTPCookieProcessor(self._cookies)]
if get_setting("use_public_dns", bool):
handlers.append(MyHTTPHandler)
if proxy['enabled']:
if proxy['use_type'] == 0 and proxy_url:
log.debug("Setting proxy from Elementum: %s" % (proxy_url))
handlers.append(parse_proxy_url(proxy_url))
elif proxy['use_type'] == 1:
log.debug("Setting proxy with custom settings: %s" %
(repr(proxy)))
handlers.append(
SocksiPyHandler(proxytype=proxy['type'],
proxyaddr=proxy['host'],
proxyport=int(proxy['port']),
username=proxy['login'],
password=proxy['password'],
rdns=True))
elif proxy['use_type'] == 2:
try:
handlers.append(antizapret.AntizapretProxyHandler())
except Exception as e:
log.info("Could not create antizapret configuration: %s" %
(e))
opener = urllib2.build_opener(*handlers)
req.add_header('User-Agent', self.user_agent)
req.add_header('Content-Language', language)
req.add_header("Accept-Encoding", "gzip")
req.add_header("Origin", url)
req.add_header("Referer", url)
if headers:
for key, value in headers.iteritems():
if value:
req.add_header(key, value)
else:
del req.headers[key.capitalize()]
if self.token:
req.add_header("Authorization", self.token)
try:
self._good_spider()
with closing(opener.open(req)) as response:
self.headers = response.headers
self._save_cookies()
if response.headers.get("Content-Encoding", "") == "gzip":
import zlib
self.content = zlib.decompressobj(
16 + zlib.MAX_WBITS).decompress(response.read())
else:
self.content = response.read()
charset = response.headers.getparam('charset')
if not charset:
match = re.search(
"""<meta(?!\s*(?:name|value)\s*=)[^>]*?charset\s*=[\s"']*([^\s"'/>]*)""",
self.content)
if match:
charset = match.group(1)
# We try to remove non-utf chars. Should we?
if (charset and charset.lower() == 'utf-8') or charset is None:
charset = 'utf-8-sig' # Changing to utf-8-sig to remove BOM if found on decode from utf-8
if charset:
log.debug('Decoding charset from %s for %s' %
(charset, repr(url)))
self.content = self.content.decode(charset, 'replace')
self.status = response.getcode()
result = True
except urllib2.HTTPError as e:
self.status = e.code
log.warning("Status for %s : %s" % (repr(url), str(self.status)))
if e.code == 403 or e.code == 503:
log.warning("CloudFlared at %s, try enabling CloudHole" % url)
except urllib2.URLError as e:
self.status = repr(e.reason)
log.warning("Status for %s : %s" % (repr(url), self.status))
except Exception as e:
import traceback
log.error("%s failed with %s:" % (repr(url), repr(e)))
map(log.debug, traceback.format_exc().split("\n"))
log.debug("Status for %s : %s" % (repr(url), str(self.status)))
return result
def process(provider, generator, filtering, has_special, verify_name=True, verify_size=True, skip_auth=False, start_time=None, timeout=None):
""" Method for processing provider results using its generator and Filtering class instance
Args:
provider (str): Provider ID
generator (function): Generator method, can be either ``extract_torrents`` or ``extract_from_api``
filtering (Filtering): Filtering class instance
has_special (bool): Whether title contains special chars
verify_name (bool): Whether to double-check the results' names match the query or not
verify_size (bool): Whether to check the results' file sizes
"""
log.debug("[%s] execute_process for %s with %s" % (provider, provider, repr(generator)))
definition = definitions[provider]
definition = get_alias(definition, get_setting("%s_alias" % provider))
client = Client(info=filtering.info, request_charset=definition['charset'], response_charset=definition['response_charset'])
token = None
logged_in = False
token_auth = False
if get_setting('kodi_language', bool):
kodi_language = xbmc.getLanguage(xbmc.ISO_639_1)
if kodi_language:
filtering.kodi_language = kodi_language
language_exceptions = get_setting('language_exceptions')
if language_exceptions.strip().lower():
filtering.language_exceptions = re.split(r',\s?', language_exceptions)
log.debug("[%s] Queries: %s" % (provider, filtering.queries))
log.debug("[%s] Extras: %s" % (provider, filtering.extras))
for query, extra in zip(filtering.queries, filtering.extras):
log.debug("[%s] Before keywords - Query: %s - Extra: %s" % (provider, repr(query), repr(extra)))
if has_special:
# Removing quotes, surrounding {title*} keywords, when title contains special chars
query = re.sub("[\"']({title.*?})[\"']", '\\1', query)
query = filtering.process_keywords(provider, query)
extra = filtering.process_keywords(provider, extra)
if not query:
continue
elif extra == '-' and filtering.results:
continue
elif start_time and timeout and time.time() - start_time + 3 >= timeout:
# Stop doing requests if there is 3 seconds left for the overall task
continue
try:
if 'charset' in definition and definition['charset'] and 'utf' not in definition['charset'].lower():
query = quote(query.encode(definition['charset']))
extra = quote(extra.encode(definition['charset']))
else:
query = quote(py2_encode(query))
extra = quote(py2_encode(extra))
except Exception as e:
log.debug("[%s] Could not quote the query (%s): %s" % (provider, query, e))
pass
log.debug("[%s] After keywords - Query: %s - Extra: %s" % (provider, repr(query), repr(extra)))
if not query:
return filtering.results
url_search = filtering.url.replace('QUERY', query)
if extra and extra != '-':
url_search = url_search.replace('EXTRA', extra)
else:
url_search = url_search.replace('EXTRA', '')
url_search = url_search.replace(' ', definition['separator'])
if definition['separator'] != '%20':
url_search = url_search.replace('%20', definition['separator'])
# MagnetDL fix...
url_search = url_search.replace('FIRSTLETTER', query[:1])
# Creating the payload for POST method
if 'post_data' in definition and not filtering.post_data:
filtering.post_data = eval(definition['post_data'])
payload = dict()
for key, value in iteritems(filtering.post_data):
if 'QUERY' in value:
payload[key] = filtering.post_data[key].replace('QUERY', query)
else:
payload[key] = filtering.post_data[key]
payload[key] = urllib.unquote(payload[key])
# Creating the payload for GET method
headers = None
data = None
if filtering.get_data:
data = dict()
for key, value in iteritems(filtering.get_data):
if 'QUERY' in value:
data[key] = filtering.get_data[key].replace('QUERY', query)
else:
data[key] = filtering.get_data[key]
log.debug("- %s query: %s" % (provider, repr(query)))
log.debug("-- %s url_search before token: %s" % (provider, repr(url_search)))
log.debug("--- %s using POST payload: %s" % (provider, repr(payload)))
log.debug("----%s filtering with post_data: %s" % (provider, repr(filtering.post_data)))
# Set search's "title" in filtering to double-check results' names
if 'filter_title' in definition and definition['filter_title']:
filtering.filter_title = True
filtering.title = query
if 'initial_url' in definition and definition['initial_url']:
url = definition['initial_url']
if not url.startswith('http'):
url = definition['root_url'] + url
client.open(url)
if token:
log.info('[%s] Reusing existing token' % provider)
url_search = url_search.replace('TOKEN', token)
elif 'token' in definition:
token_url = definition['base_url'] + definition['token']
log.debug("[%s] Getting token for %s at %s" % (provider, provider, repr(token_url)))
client.open(py2_encode(token_url))
try:
token_data = json.loads(client.content)
except:
log.error('%s: Failed to get token for %s' % (provider, repr(url_search)))
return filtering.results
log.debug("[%s] Token response for %s: %s" % (provider, provider, repr(token_data)))
if 'token' in token_data:
token = token_data['token']
log.debug("[%s] Got token for %s: %s" % (provider, provider, repr(token)))
url_search = url_search.replace('TOKEN', token)
else:
log.warning('%s: Unable to get token for %s' % (provider, repr(url_search)))
if logged_in:
log.info("[%s] Reusing previous login" % provider)
elif token_auth:
log.info("[%s] Reusing previous token authorization" % provider)
elif 'private' in definition and definition['private']:
username = get_setting('%s_username' % provider, unicode)
password = get_setting('%s_password' % provider, unicode)
passkey = get_setting('%s_passkey' % provider, unicode)
if not username and not password and not passkey:
for addon_name in ('script.magnetic.%s' % provider, 'script.magnetic.%s-mc' % provider):
for setting in ('username', 'password'):
try:
value = xbmcaddon.Addon(addon_name).getSetting(setting)
set_setting('%s_%s' % (provider, setting), value)
if setting == 'username':
username = value
if setting == 'password':
password = value
except:
pass
if username:
client.username = username
url_search = url_search.replace('USERNAME', username)
if passkey:
logged_in = True
client.passkey = passkey
url_search = url_search.replace('PASSKEY', passkey)
elif 'login_object' in definition and definition['login_object']:
login_object = None
login_headers = None
logged_in = skip_auth
try:
login_object = definition['login_object'].replace('USERNAME', 'u"%s"' % username).replace('PASSWORD', 'u"%s"' % password)
except Exception as e:
log.error("Could not make login object for %s: %s" % (provider, e))
try:
if 'login_headers' in definition and definition['login_headers']:
login_headers = eval(definition['login_headers'])
except Exception as e:
log.error("Could not make login headers for %s: %s" % (provider, e))
# TODO generic flags in definitions for those...
if 'csrf_token' in definition and definition['csrf_token']:
client.open(definition['root_url'] + definition['login_path'])
if client.content:
csrf_token = re.search(r'name=\"_?csrf_token\" value=\"(.*?)\"', client.content)
if csrf_token:
login_object = login_object.replace('CSRF_TOKEN', '"%s"' % csrf_token.group(1))
else:
logged_in = True
if 'token_auth' in definition:
# log.debug("[%s] logging in with: %s" % (provider, login_object))
if client.open(definition['root_url'] + definition['token_auth'], post_data=eval(login_object)):
try:
token_data = json.loads(client.content)
except:
log.error('%s: Failed to get token from %s' % (provider, definition['token_auth']))
return filtering.results
log.debug("[%s] Token response for %s: %s" % (provider, provider, repr(token_data)))
if 'token' in token_data:
client.token = token_data['token']
log.debug("[%s] Auth token for %s: %s" % (provider, provider, repr(client.token)))
else:
log.error('[%s] Unable to get auth token for %s' % (provider, repr(url_search)))
return filtering.results
log.info('[%s] Token auth successful' % provider)
token_auth = True
else:
log.error("[%s] Token auth failed with response: %s" % (provider, repr(client.content)))
return filtering.results
elif not logged_in and client.login(definition['root_url'], definition['login_path'],
eval(login_object), login_headers, definition['login_failed']):
log.info('[%s] Login successful' % provider)
logged_in = True
elif not logged_in:
log.error("[%s] Login failed: %s", provider, client.status)
log.debug("[%s] Failed login content: %s", provider, repr(client.content))
return filtering.results
if logged_in:
if provider == 'hd-torrents':
client.open(definition['root_url'] + '/torrents.php')
csrf_token = re.search(r'name="csrfToken" value="(.*?)"', client.content)
url_search = url_search.replace("CSRF_TOKEN", csrf_token.group(1))
client.save_cookies()
log.info("[%s] > %s search URL: %s" % (provider, definition['name'].rjust(longest), url_search))
if 'headers' in definition and definition['headers']:
headers = eval(definition['headers'])
log.info("[%s] > %s headers: %s" % (provider, definition['name'].rjust(longest), headers))
client.open(py2_encode(url_search), post_data=payload, get_data=data, headers=headers)
filtering.results.extend(
generate_payload(provider,
generator(provider, client),
filtering,
verify_name,
verify_size))
return filtering.results
def config():
global _config
if not _config and CACHE_DIR:
with shelf("antizapret.pac_config", ttl=CACHE) as pac:
if not pac.get("value"):
data = None
log.info("Fetching Antizapret PAC file")
try:
req = urllib2.Request(PAC_URL,
headers={
'User-Agent': USER_AGENT,
'Accept-Encoding': 'gzip',
'Origin': PAC_URL,
'Referer': PAC_URL
})
response = urllib2.urlopen(req)
if response.headers.get("Content-Encoding", "") == "gzip":
import zlib
data = zlib.decompressobj(16 +
zlib.MAX_WBITS).decompress(
response.read())
else:
data = response.read()
except Exception as e:
log.error("Fetching Antizapret PAC failed with code: %s" %
(repr(e)))
data = ""
pac["value"] = {
"servers_usual": [],
"servers_special": [],
"domains": [],
"specials": [],
"ips": [],
"dn": [],
}
# Find servers for specific cases
r_usual = re.search(
r'if \(yip === 1 \|\| shost === curarr\[i\]\) \{.*?return "(.*?)".*?\}',
data,
flags=re.DOTALL)
r_special = re.search(
r'if \(isInNet\(oip, special\[i\]\[0\], special\[i\]\[1\]\)\) \{return "(.*?)";\}',
data,
flags=re.DOTALL)
# Find arrays containing domains
res = re.findall(r'(d_\w+) = "(.*?)"', data, flags=re.DOTALL)
r_list_special = re.search(r'special = \[(.*?)\];',
data,
flags=re.DOTALL)
r_dn = re.search(r'dn = \{(.*?)\};', data, flags=re.DOTALL)
if r_usual:
pac["value"]["servers_usual"] = get_servers(
r_usual.group(1))
if r_special:
pac["value"]["servers_special"] = get_servers(
r_special.group(1))
if r_dn:
ary = r_dn.group(1).replace("'", "").split(", ")
for i in ary:
# ary2 = i.split(":")
# pac["value"]["dn"].append([i[1], i[0]])
pac["value"]["dn"].append(i.split(":"))
if r_list_special:
for i in r_list_special.group(1).replace(
'],[', '] , [').replace('"', '').split(" , "):
s = i.replace('[', '').replace(']', '').rstrip(',')
ary = s.strip().rstrip(',').split(', ')
pac["value"]["specials"].append([ary[0], int(ary[1])])
if res:
domains = ""
ips = ""
for r in res:
l = r[1].replace('\\', '')
if r[0] == 'd_ipaddr':
ips += l
else:
domains += l
pac["value"]["domains"] = domains.split(" ")
pac["value"]["ips"] = re.findall(r'.{8}',
ips,
flags=re.DOTALL)
_config = pac["value"]
return _config
