def download_archive_and_add_subtitle_files(self, link, language, video,
fps, num_cds):
logger.info('Downloading subtitle %r', link)
cache_key = sha1(link.encode("utf-8")).digest()
request = region.get(cache_key)
if request is NO_VALUE:
request = self.session.get(
link, headers={'Referer': 'https://subsunacs.net/search.php'})
request.raise_for_status()
region.set(cache_key, request)
else:
logger.info('Cache file: %s',
codecs.encode(cache_key, 'hex_codec').decode('utf-8'))
try:
archive_stream = io.BytesIO(request.content)
if is_rarfile(archive_stream):
return self.process_archive_subtitle_files(
RarFile(archive_stream), language, video, link, fps,
num_cds)
elif is_zipfile(archive_stream):
return self.process_archive_subtitle_files(
ZipFile(archive_stream), language, video, link, fps,
num_cds)
elif archive_stream.seek(0) == 0 and is_7zfile(archive_stream):
return self.process_archive_subtitle_files(
SevenZipFile(archive_stream), language, video, link, fps,
num_cds)
except:
pass
logger.error('Ignore unsupported archive %r', request.headers)
region.delete(cache_key)
return []
def request(self, method, url, *args, **kwargs):
parsed_url = urlparse(url)
domain = parsed_url.netloc
cache_key = "cf_data3_%s" % domain
if not self.cookies.get("cf_clearance", "", domain=domain):
cf_data = region.get(cache_key)
if cf_data is not NO_VALUE:
cf_cookies, hdrs = cf_data
logger.debug("Trying to use old cf data for %s: %s", domain, cf_data)
for cookie, value in cf_cookies.iteritems():
self.cookies.set(cookie, value, domain=domain)
self.headers = hdrs
ret = self._request(method, url, *args, **kwargs)
try:
cf_data = self.get_cf_live_tokens(domain)
except:
pass
else:
if cf_data and "cf_clearance" in cf_data[0] and cf_data[0]["cf_clearance"]:
if cf_data != region.get(cache_key):
logger.debug("Storing cf data for %s: %s", domain, cf_data)
region.set(cache_key, cf_data)
elif cf_data[0]["cf_clearance"]:
logger.debug("CF Live tokens not updated")
return ret
def download_archive_and_add_subtitle_files(self, link, language, video, fps, subs_id):
logger.info('Downloading subtitle %r', link)
cache_key = sha1(link.encode("utf-8")).digest()
request = region.get(cache_key)
if request is NO_VALUE:
time.sleep(1)
request = self.retry(self.session.post(link, data={
'id': subs_id,
'lng': language.basename.upper()
}, headers={
'referer': link
}, allow_redirects=False))
if not request:
return []
request.raise_for_status()
region.set(cache_key, request)
else:
logger.info('Cache file: %s', codecs.encode(cache_key, 'hex_codec').decode('utf-8'))
try:
archive_stream = io.BytesIO(request.content)
if is_rarfile(archive_stream):
return self.process_archive_subtitle_files(RarFile(archive_stream), language, video, link, fps, subs_id)
elif is_zipfile(archive_stream):
return self.process_archive_subtitle_files(ZipFile(archive_stream), language, video, link, fps, subs_id)
except:
pass
logger.error('Ignore unsupported archive %r', request.headers)
region.delete(cache_key)
return []
def request(self, method, url, *args, **kwargs):
parsed_url = urlparse(url)
domain = parsed_url.netloc
cache_key = "cf_data2_%s" % domain
if not self.cookies.get("cf_clearance", "", domain=domain):
cf_data = region.get(cache_key)
if cf_data is not NO_VALUE:
cf_cookies, user_agent, hdrs = cf_data
logger.debug("Trying to use old cf data for %s: %s", domain, cf_data)
for cookie, value in cf_cookies.iteritems():
self.cookies.set(cookie, value, domain=domain)
self._hdrs = hdrs
self._ua = user_agent
self.headers['User-Agent'] = self._ua
ret = super(CFSession, self).request(method, url, *args, **kwargs)
if self._was_cf:
self._was_cf = False
logger.debug("We've hit CF, trying to store previous data")
try:
cf_data = self.get_cf_live_tokens(domain)
except:
logger.debug("Couldn't get CF live tokens for re-use. Cookies: %r", self.cookies)
pass
else:
if cf_data != region.get(cache_key) and cf_data[0]["cf_clearance"]:
logger.debug("Storing cf data for %s: %s", domain, cf_data)
region.set(cache_key, cf_data)
return ret
def request(self, method, url, *args, **kwargs):
parsed_url = urlparse(url)
domain = parsed_url.netloc
cache_key = "cf_data_%s" % domain
if not self.cookies.get("__cfduid", "", domain=domain):
cf_data = region.get(cache_key)
if cf_data is not NO_VALUE:
cf_cookies, user_agent = cf_data
logger.debug("Trying to use old cf data for %s: %s", domain, cf_data)
for cookie, value in cf_cookies.iteritems():
self.cookies.set(cookie, value, domain=domain)
self.headers['User-Agent'] = user_agent
ret = super(CFSession, self).request(method, url, *args, **kwargs)
try:
cf_data = self.get_live_tokens(domain)
except:
pass
else:
if cf_data != region.get(cache_key) and self.cookies.get("__cfduid", "", domain=domain)\
and self.cookies.get("cf_clearance", "", domain=domain):
logger.debug("Storing cf data for %s: %s", domain, cf_data)
region.set(cache_key, cf_data)
return ret
def _create_filters(self, languages):
self.filters = dict(HearingImpaired="2")
acc_filters = self.filters.copy()
if self.only_foreign:
self.filters["ForeignOnly"] = "True"
acc_filters["ForeignOnly"] = self.filters["ForeignOnly"].lower()
logger.info("Only searching for foreign/forced subtitles")
selected_ids = []
for l in languages:
lid = language_ids.get(l.basename,
language_ids.get(l.alpha3, None))
if lid:
selected_ids.append(str(lid))
acc_filters["SelectedIds"] = selected_ids
self.filters["LanguageFilter"] = ",".join(acc_filters["SelectedIds"])
last_filters = region.get("subscene_filters")
if last_filters != acc_filters:
region.set("subscene_filters", acc_filters)
logger.debug("Setting account filters to %r", acc_filters)
self.session.post("https://u.subscene.com/filter",
acc_filters,
allow_redirects=False)
logger.debug("Filter created: '%s'" % self.filters)
def login(self):
try:
r = self.session.post(self.server_url + 'login',
json={"username": self.username, "password": self.password},
allow_redirects=False,
timeout=10)
except (ConnectionError, Timeout, ReadTimeout):
raise ServiceUnavailable('Unknown Error, empty response: %s: %r' % (r.status_code, r))
else:
if r.status_code == 200:
try:
self.token = r.json()['token']
except ValueError:
raise ProviderError('Invalid JSON returned by provider')
else:
self.session.headers.update({'Authorization': 'Beaker ' + self.token})
region.set("oscom_token", self.token)
return True
elif r.status_code == 401:
raise AuthenticationError('Login failed: {}'.format(r.reason))
elif r.status_code == 429:
raise TooManyRequests()
else:
raise ProviderError('Bad status code: {}'.format(r.status_code))
finally:
return False
def request(self, method, url, *args, **kwargs):
parsed_url = urlparse(url)
domain = parsed_url.netloc
cache_key = "cf_data3_%s" % domain
if not self.cookies.get("cf_clearance", "", domain=domain):
cf_data = region.get(cache_key)
if cf_data is not NO_VALUE:
cf_cookies, hdrs = cf_data
logger.debug("Trying to use old cf data for %s: %s", domain,
cf_data)
for cookie, value in six.iteritems(cf_cookies):
self.cookies.set(cookie, value, domain=domain)
self.headers = hdrs
ret = self._request(method, url, *args, **kwargs)
try:
cf_data = self.get_cf_live_tokens(domain)
except:
pass
else:
if cf_data and "cf_clearance" in cf_data[0] and cf_data[0][
"cf_clearance"]:
if cf_data != region.get(cache_key):
logger.debug("Storing cf data for %s: %s", domain, cf_data)
region.set(cache_key, cf_data)
elif cf_data[0]["cf_clearance"]:
logger.debug("CF Live tokens not updated")
return ret
def login(self):
logger.debug('Legendasdivx.pt :: Logging in')
try:
# sleep for a 1 second before another request
sleep(1)
res = self.session.get(self.loginpage)
res.raise_for_status()
bsoup = ParserBeautifulSoup(res.content, ['lxml'])
_allinputs = bsoup.findAll('input')
data = {}
# necessary to set 'sid' for POST request
for field in _allinputs:
data[field.get('name')] = field.get('value')
# sleep for a 1 second before another request
sleep(1)
data['username'] = self.username
data['password'] = self.password
res = self.session.post(self.loginpage, data)
res.raise_for_status()
# make sure we're logged in
logger.debug(
'Legendasdivx.pt :: Logged in successfully: PHPSESSID: %s',
self.session.cookies.get_dict()['PHPSESSID'])
cj = self.session.cookies.copy()
store_cks = ("PHPSESSID", "phpbb3_2z8zs_sid", "phpbb3_2z8zs_k",
"phpbb3_2z8zs_u", "lang")
for cn in iter(self.session.cookies.keys()):
if cn not in store_cks:
del cj[cn]
# store session cookies on cache
logger.debug(
"Legendasdivx.pt :: Storing legendasdivx session cookies: %r",
cj)
region.set("legendasdivx_cookies2", cj)
except KeyError:
logger.error(
"Legendasdivx.pt :: Couldn't get session ID, check your credentials"
)
raise AuthenticationError(
"Legendasdivx.pt :: Couldn't get session ID, check your credentials"
)
except HTTPError as e:
if "bloqueado" in res.text.lower():
logger.error(
"LegendasDivx.pt :: Your IP is blocked on this server.")
raise IPAddressBlocked(
"LegendasDivx.pt :: Your IP is blocked on this server.")
logger.error("Legendasdivx.pt :: HTTP Error %s", e)
raise TooManyRequests("Legendasdivx.pt :: HTTP Error %s", e)
except Exception as e:
logger.error("LegendasDivx.pt :: Uncaught error: %r", e)
raise ServiceUnavailable("LegendasDivx.pt :: Uncaught error: %r",
e)
def initialize(self):
self.session = Session()
self.session.headers[
'User-Agent'] = 'Subliminal/%s' % subliminal.__short_version__
if self.USE_ADDICTED_RANDOM_AGENTS:
from .utils import FIRST_THOUSAND_OR_SO_USER_AGENTS as AGENT_LIST
logger.debug("Addic7ed: using random user agents")
self.session.headers['User-Agent'] = AGENT_LIST[randint(
0,
len(AGENT_LIST) - 1)]
self.session.headers['Referer'] = self.server_url
# login
if self.username and self.password:
ccks = region.get("addic7ed_cookies", expiration_time=86400)
if ccks != NO_VALUE:
try:
self.session.cookies._cookies.update(ccks)
r = self.session.get(self.server_url + 'panel.php',
allow_redirects=False,
timeout=10)
if r.status_code == 302:
logger.info('Addic7ed: Login expired')
region.delete("addic7ed_cookies")
else:
logger.info('Addic7ed: Reusing old login')
self.logged_in = True
return
except:
pass
logger.info('Addic7ed: Logging in')
data = {
'username': self.username,
'password': self.password,
'Submit': 'Log in'
}
r = self.session.post(
self.server_url + 'dologin.php',
data,
allow_redirects=False,
timeout=10,
headers={"Referer": self.server_url + "login.php"})
if "relax, slow down" in r.content:
raise TooManyRequests(self.username)
if r.status_code != 302:
raise AuthenticationError(self.username)
region.set("addic7ed_cookies", self.session.cookies._cookies)
logger.debug('Addic7ed: Logged in')
self.logged_in = True
def login(self):
r = self.session.get("https://subscene.com/account/login")
if "Server Error" in r.text:
logger.error("Login unavailable; Maintenance?")
raise ServiceUnavailable("Login unavailable; Maintenance?")
match = re.search(
r"<script id='modelJson' type='application/json'>\s*(.+)\s*</script>",
r.text)
if match:
h = six.moves.html_parser.HTMLParser()
data = json.loads(h.unescape(match.group(1)))
login_url = six.moves.urllib.parse.urljoin(data["siteUrl"],
data["loginUrl"])
time.sleep(1.0)
r = self.session.post(
login_url, {
"username": self.username,
"password": self.password,
data["antiForgery"]["name"]: data["antiForgery"]["value"]
})
pep_content = re.search(
r"<form method=\"post\" action=\"https://subscene\.com/\">"
r".+name=\"id_token\".+?value=\"(?P<id_token>.+?)\".*?"
r"access_token\".+?value=\"(?P<access_token>.+?)\".+?"
r"token_type.+?value=\"(?P<token_type>.+?)\".+?"
r"expires_in.+?value=\"(?P<expires_in>.+?)\".+?"
r"scope.+?value=\"(?P<scope>.+?)\".+?"
r"state.+?value=\"(?P<state>.+?)\".+?"
r"session_state.+?value=\"(?P<session_state>.+?)\"", r.text,
re.MULTILINE | re.DOTALL)
if pep_content:
r = self.session.post(SITE_DOMAIN, pep_content.groupdict())
try:
r.raise_for_status()
except Exception:
raise ProviderError(
"Something went wrong when trying to log in: %s",
traceback.format_exc())
else:
cj = self.session.cookies.copy()
store_cks = ("scene", "idsrv", "idsrv.xsrf",
"idsvr.clients", "idsvr.session",
"idsvr.username")
for cn in six.iterkeys(self.session.cookies):
if cn not in store_cks:
del cj[cn]
logger.debug("Storing cookies: %r", cj)
region.set("subscene_cookies2", cj)
return
raise ProviderError("Something went wrong when trying to log in #1")
def find_endpoint(session, content=None):
endpoint = region.get("subscene_endpoint2")
if endpoint is NO_VALUE:
if not content:
content = session.get(SITE_DOMAIN).text
m = ENDPOINT_RE.search(content)
if m:
endpoint = m.group(1).strip()
logger.debug("Switching main endpoint to %s", endpoint)
region.set("subscene_endpoint2", endpoint)
return endpoint
def _get_show_ids(self):
"""Get the ``dict`` of show ids per series by querying the `shows.php` page.
:return: show id per series, lower case and without quotes.
:rtype: dict
# patch: add punctuation cleaning
"""
# get the show page
logger.info('Getting show ids')
region.set(self.last_show_ids_fetch_key, datetime.datetime.now())
r = self.session.get(self.server_url + 'shows.php', timeout=10)
r.raise_for_status()
# LXML parser seems to fail when parsing Addic7ed.com HTML markup.
# Last known version to work properly is 3.6.4 (next version, 3.7.0, fails)
# Assuming the site's markup is bad, and stripping it down to only contain what's needed.
show_cells = re.findall(show_cells_re, r.content)
if show_cells:
soup = ParserBeautifulSoup(
b''.join(show_cells).decode('utf-8', 'ignore'),
['lxml', 'html.parser'])
else:
# If RegEx fails, fall back to original r.text and use 'html.parser'
soup = ParserBeautifulSoup(r.text, ['html.parser'])
# populate the show ids
show_ids = {}
shows = soup.select('td > h3 > a[href^="/show/"]')
for show in shows:
show_clean = sanitize(show.text,
default_characters=self.sanitize_characters)
try:
show_id = int(show['href'][6:])
except ValueError:
continue
show_ids[show_clean] = show_id
match = series_year_re.match(show_clean)
if match and match.group(2) and match.group(1) not in show_ids:
# year found, also add it without year
show_ids[match.group(1)] = show_id
soup.decompose()
soup = None
logger.debug('Found %d show ids', len(show_ids))
if not show_ids:
raise Exception("Addic7ed: No show IDs found!")
return show_ids
def request(self, host, handler, request_body, verbose=0):
"""
Make an xmlrpc request.
"""
url = self._build_url(host, handler)
cache_key = "xm%s_%s" % (self.xm_ver, host)
old_sessvar = self.session.cookies.get(self.session_var, "")
if not old_sessvar:
data = region.get(cache_key)
if data is not NO_VALUE:
logger.debug("Trying to re-use headers/cookies for %s" % host)
self.session.cookies, self.session.headers = data
old_sessvar = self.session.cookies.get(self.session_var, "")
try:
resp = self.session.post(url, data=request_body,
stream=True, timeout=self.timeout, proxies=self.proxies,
verify=self.verify)
if self.session_var in resp.cookies and resp.cookies[self.session_var] != old_sessvar:
logger.debug("Storing %s cookies" % host)
region.set(cache_key, [self.session.cookies, self.session.headers])
except ValueError:
logger.debug("Wiping cookies/headers cache (VE) for %s" % host)
region.delete(cache_key)
raise
except Exception:
logger.debug("Wiping cookies/headers cache (EX) for %s" % host)
region.delete(cache_key)
raise # something went wrong
else:
try:
resp.raise_for_status()
except requests.exceptions.HTTPError:
logger.debug("Wiping cookies/headers cache (RE) for %s" % host)
region.delete(cache_key)
raise
try:
if 'x-ratelimit-remaining' in resp.headers and int(resp.headers['x-ratelimit-remaining']) <= 2:
raise APIThrottled()
except ValueError:
logger.info('Couldn\'t parse "x-ratelimit-remaining": %r' % resp.headers['x-ratelimit-remaining'])
self.verbose = verbose
try:
return self.parse_response(resp.raw)
except:
logger.debug("Bad response data: %r", resp.raw)
def log_in(self, server_url=None):
if server_url:
self.terminate()
self.server = self.get_server_proxy(server_url)
response = self.retry(lambda: checked(
self.server.LogIn(self.username, self.password, 'eng',
os.environ.get("SZ_USER_AGENT", "Sub-Zero/2"))))
self.token = response['token']
logger.debug('Logged in with token %r', self.token)
region.set("os_token", self.token)
def download_subtitle(self, subtitle):
last_dls = region.get("addic7ed_dls")
now = datetime.datetime.now()
one_day = datetime.timedelta(hours=24)
def raise_limit():
logger.info("Addic7ed: Downloads per day exceeded (%s)", cap)
raise DownloadLimitPerDayExceeded
if not isinstance(last_dls, types.ListType):
last_dls = []
else:
# filter all non-expired DLs
last_dls = filter(lambda t: t + one_day > now, last_dls)
region.set("addic7ed_dls", last_dls)
cap = self.vip and 80 or 40
amount = len(last_dls)
if amount >= cap:
raise_limit()
# download the subtitle
r = self.session.get(self.server_url + subtitle.download_link,
headers={'Referer': subtitle.page_link},
timeout=10)
r.raise_for_status()
if r.status_code == 304:
raise TooManyRequests()
if not r.content:
# Provider wrongful return a status of 304 Not Modified with an empty content
# raise_for_status won't raise exception for that status code
logger.error(
'Addic7ed: Unable to download subtitle. No data returned from provider'
)
return
# detect download limit exceeded
if r.headers['Content-Type'] == 'text/html':
raise DownloadLimitExceeded
subtitle.content = fix_line_ending(r.content)
last_dls.append(datetime.datetime.now())
region.set("addic7ed_dls", last_dls)
logger.info("Addic7ed: Used %s/%s downloads", amount + 1, cap)
if amount + 1 >= cap:
raise_limit()
def log_in_url(self, server_url):
self.token = None
self.server = self.get_server_proxy(server_url)
response = self.retry(lambda: checked(lambda: self.server.LogIn(
self.username, self.password, 'eng',
os.environ.get("SZ_USER_AGENT", "Sub-Zero/2"))))
self.token = response['token']
logger.debug('Logged in with token %r',
self.token[:10] + "X" * (len(self.token) - 10))
region.set("os_token", bytearray(self.token, encoding='utf-8'))
region.set("os_server_url", bytearray(server_url, encoding='utf-8'))
def login(self):
r = self.retry(lambda: checked(lambda: self.session.post(
self.server_url + 'login',
json={
"username": self.username,
"password": self.password
},
allow_redirects=False,
timeout=30),
validate_json=True,
json_key_name='token'),
amount=retry_amount)
self.token = r.json()['token']
region.set("oscom_token", self.token)
return
def log_in(self, server_url=None):
if server_url:
self.terminate()
self.server = self.get_server_proxy(server_url)
response = self.retry(
lambda: checked(
lambda: self.server.LogIn(self.username, self.password, 'eng',
os.environ.get("SZ_USER_AGENT", "Sub-Zero/2"))
)
)
self.token = response['token']
logger.debug('Logged in with token %r', self.token[:10]+"X"*(len(self.token)-10))
region.set("os_token", self.token)
def log_in(self, server_url=None):
if server_url:
self.terminate()
self.server = self.get_server_proxy(server_url)
response = self.retry(
lambda: checked(
lambda: self.server.LogIn(self.username, self.password, 'eng',
"Bazarr/1")
)
)
self.token = response['token']
logger.debug('Logged in with token %r', self.token[:10]+"X"*(len(self.token)-10))
region.set("os_token", self.token)
def _get_show_ids(self):
"""Get the ``dict`` of show ids per series by querying the `shows.php` page.
:return: show id per series, lower case and without quotes.
:rtype: dict
# patch: add punctuation cleaning
"""
# get the show page
logger.info('Getting show ids')
region.set(self.last_show_ids_fetch_key, datetime.datetime.now())
r = self.session.get(self.server_url, timeout=10)
r.raise_for_status()
soup = ParserBeautifulSoup(r.content.decode('utf-8', 'ignore'), ['lxml', 'html.parser'])
# populate the show ids
show_ids = {}
shows = soup.find(id='qsShow')
for show in shows:
if hasattr(show, 'attrs'):
try:
show_id = int(show.attrs['value'])
except ValueError:
continue
if show_id != 0:
show_clean = sanitize(show.text, default_characters=self.sanitize_characters)
show_ids[show_clean] = show_id
match = series_year_re.match(show_clean)
if match and match.group(2) and match.group(1) not in show_ids:
# year found, also add it without year
show_ids[match.group(1)] = show_id
soup.decompose()
soup = None
logger.debug('Found %d show ids', len(show_ids))
if not show_ids:
raise Exception("Addic7ed: No show IDs found!")
return show_ids
def fill_addic7ed_show_id_cache():
"""
Fill the Addic7ed show_id cache used by the Addic7ed provider with our Addic7ed show name mappings.
If the show name of a parsed file is not found on Addic7ed, no subtitles can be search on Addic7ed.
That's why we need to force the id in the subliminal cache, in order to find the subtitles.
Subtitles on Addic7ed can only be listed by their own internal Addic7ed id.
Filling of the cache will make sure that this method of addic7ed.py always returns our mapping for a show_name.
Since we are filling it before each run, the expiration_time will never be met and we'll always use mapping id.
If there is no custom mapping, a search will be done on Addic7ed with the show_name.
>>>@region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME)
>>>def _search_show_id(self, series, year=None):
"""
for x in autosubliminal.ADDIC7EDSHOWNAMEMAPPING:
# Dogpile cache expects native strings as keys!
cache_value = int(autosubliminal.ADDIC7EDSHOWNAMEMAPPING[x])
cache_key = s2n(ADDIC7ED_SEARCH_SHOW_ID_CACHE_PREFIX + '|' + x)
region.set(cache_key, cache_value)
cache_key = s2n(CUSTOM_ADDIC7ED_SEARCH_SHOW_ID_CACHE_PREFIX + '|' + x)
region.set(cache_key, cache_value)
def search(term,
release=True,
session=None,
year=None,
limit_to=SearchTypes.Exact,
throttle=0):
# note to subscene: if you actually start to randomize the endpoint, we'll have to query your server even more
if release:
endpoint = "release"
else:
endpoint = region.get("subscene_endpoint2")
if endpoint is NO_VALUE:
ret = session.get(SITE_DOMAIN)
time.sleep(throttle)
m = ENDPOINT_RE.search(ret.text)
if m:
endpoint = m.group(1).strip()
logger.debug("Switching main endpoint to %s", endpoint)
region.set("subscene_endpoint2", endpoint)
soup = soup_for("%s/subtitles/%s" % (SITE_DOMAIN, endpoint),
data={"query": term},
session=session)
if soup:
if "Subtitle search by" in str(soup):
rows = soup.find("table").tbody.find_all("tr")
subtitles = Subtitle.from_rows(rows)
return Film(term, subtitles=subtitles)
for junk, search_type in SearchTypes.__members__.items():
if section_exists(soup, search_type):
return get_first_film(soup,
search_type,
year=year,
session=session)
if limit_to == search_type:
return
def request(self, method, url, *args, **kwargs):
parsed_url = urlparse(url)
domain = parsed_url.netloc
cache_key = "cf_data2_%s" % domain
if not self.cookies.get("cf_clearance", "", domain=domain):
cf_data = region.get(cache_key)
if cf_data is not NO_VALUE:
cf_cookies, user_agent, hdrs = cf_data
logger.debug("Trying to use old cf data for %s: %s", domain,
cf_data)
for cookie, value in cf_cookies.iteritems():
self.cookies.set(cookie, value, domain=domain)
self._hdrs = hdrs
self._ua = user_agent
self.headers['User-Agent'] = self._ua
ret = super(CFSession, self).request(method, url, *args, **kwargs)
if self._was_cf:
self._was_cf = False
logger.debug("We've hit CF, trying to store previous data")
try:
cf_data = self.get_cf_live_tokens(domain)
except:
logger.debug(
"Couldn't get CF live tokens for re-use. Cookies: %r",
self.cookies)
pass
else:
if cf_data != region.get(
cache_key) and cf_data[0]["cf_clearance"]:
logger.debug("Storing cf data for %s: %s", domain, cf_data)
region.set(cache_key, cf_data)
return ret
def log_in(self):
login_params = dict(username=self.username,
password=self.password,
json=True)
try:
response = self.session.post(self.api_gettoken_url,
params=login_params)
if response.status_code == request_codes.ok:
resp_json = response.json()
self.login_token = resp_json.get('Token')
self.user_id = resp_json.get('UserId')
self.token_exp = dateutil.parser.parse(
resp_json.get('ExpirationDate'))
region.set("titlovi_token",
[self.user_id, self.login_token, self.token_exp])
logger.debug('New token obtained')
elif response.status_code == request_codes.unauthorized:
raise AuthenticationError('Login failed')
except RequestException as e:
logger.error(e)
def store_verification(site_name, session):
region.set("%s_data" % site_name,
(session.cookies._cookies, session.headers["User-Agent"]))
def store_verification(site_name, session):
region.set("%s_data" % site_name, (session.cookies._cookies, session.headers["User-Agent"]))
def query(self, language, title, season=None, episode=None, year=None):
# search for titles
sanitized_title = sanitize(title)
titles = self.search_titles(sanitized_title, season)
# search for titles with the quote or dot character
ignore_characters = {'\'', '.'}
if any(c in title for c in ignore_characters):
titles.update(self.search_titles(sanitize(title, ignore_characters=ignore_characters), season))
subtitles = []
# iterate over titles
for title_id, t in titles.items():
# discard mismatches on title
sanitized_result = sanitize(t['title'])
if sanitized_result != sanitized_title:
logger.debug("Mismatched title, discarding title %d (%s)",
title_id, sanitized_result)
continue
# episode
if season and episode:
# discard mismatches on type
if t['type'] != 'episode':
logger.debug("Mismatched 'episode' type, discarding title %d (%s)", title_id, sanitized_result)
continue
# discard mismatches on season
if 'season' not in t or t['season'] != season:
logger.debug('Mismatched season %s, discarding title %d (%s)',
t.get('season'), title_id, sanitized_result)
continue
# movie
else:
# discard mismatches on type
if t['type'] != 'movie':
logger.debug("Mismatched 'movie' type, discarding title %d (%s)", title_id, sanitized_result)
continue
# discard mismatches on year
if year is not None and 'year' in t and t['year'] != year:
logger.debug("Mismatched movie year, discarding title %d (%s)", title_id, sanitized_result)
continue
# iterate over title's archives
for a in self.get_archives(title_id, language.legendastv):
# clean name of path separators and pack flags
clean_name = a.name.replace('/', '-')
if a.pack and clean_name.startswith('(p)'):
clean_name = clean_name[3:]
# guess from name
guess = guessit(clean_name, {'type': t['type']})
# episode
if season and episode:
# discard mismatches on episode in non-pack archives
if not a.pack and 'episode' in guess and guess['episode'] != episode:
logger.debug('Mismatched episode %s, discarding archive: %s',
guess['episode'], a.name)
continue
# compute an expiration time based on the archive timestamp
expiration_time = (datetime.utcnow().replace(tzinfo=pytz.utc) - a.timestamp).total_seconds()
# attempt to get the releases from the cache
cache_item = releases_key.format(archive_id=a.id, archive_name=a.name)
releases = region.get(cache_item, expiration_time=expiration_time)
# the releases are not in cache or cache is expired
if releases == NO_VALUE:
logger.info('Releases not found in cache')
# download archive
self.download_archive(a)
# extract the releases
releases = []
for name in a.content.namelist():
# discard the legendastv file
if name.startswith('Legendas.tv'):
continue
# discard hidden files
if os.path.split(name)[-1].startswith('.'):
continue
# discard non-subtitle files
if not name.lower().endswith(SUBTITLE_EXTENSIONS):
continue
releases.append(name)
# cache the releases
region.set(cache_item, releases)
# iterate over releases
for r in releases:
subtitle = LegendasTVSubtitle(language, t['type'], t['title'], t.get('year'), t.get('imdb_id'),
t.get('season'), a, r)
logger.debug('Found subtitle %r', subtitle)
subtitles.append(subtitle)
return subtitles
