class Hoster(Plugin):
__name__ = "Hoster"
__type__ = "hoster"
__version__ = "0.18"
__status__ = "testing"
__pattern__ = r'^unmatchable$'
__config__ = [] #: [("name", "type", "desc", "default")]
__description__ = """Base hoster plugin"""
__license__ = "GPLv3"
__authors__ = [("RaNaN" , "*****@*****.**" ),
("spoob" , "*****@*****.**" ),
("mkaay" , "*****@*****.**" ),
("Walter Purcaro", "*****@*****.**")]
def __init__(self, pyfile):
self._init(pyfile.m.core)
#: Engage wan reconnection
self.wantReconnect = False #@TODO: Change to `want_reconnect` in 0.4.10
#: Enable simultaneous processing of multiple downloads
self.multiDL = True #@TODO: Change to `multi_dl` in 0.4.10
self.limitDL = 0 #@TODO: Change to `limit_dl` in 0.4.10
#: time.time() + wait in seconds
self.wait_until = 0
self.waiting = False
#: Account handler instance, see :py:class:`Account`
self.account = None
self.user = None
self.req = None #: Browser instance, see `network.Browser`
#: Associated pyfile instance, see `PyFile`
self.pyfile = pyfile
self.thread = None #: Holds thread in future
#: Location where the last call to download was saved
self.last_download = ""
#: Re match of the last call to `checkDownload`
self.last_check = None
#: Js engine, see `JsEngine`
self.js = self.pyload.js
#: Captcha stuff
self.captcha = Captcha(self)
#: Some plugins store html code here
self.html = None
#: Dict of the amount of retries already made
self.retries = {}
self.retry_free = False #@TODO: Recheck in 0.4.10
self._setup()
self.init()
@classmethod
def get_info(cls, url="", html=""):
url = _fixurl(url)
url_p = urlparse.urlparse(url)
return {'name' : (url_p.path.split('/')[-1] or
url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] or
url_p.netloc.split('.', 1)[0]),
'size' : 0,
'status': 3 if url else 8,
'url' : url}
def init(self):
"""
Initialize the plugin (in addition to `__init__`)
"""
pass
def setup(self):
"""
Setup for enviroment and other things, called before downloading (possibly more than one time)
"""
pass
def _setup(self):
if self.account:
self.req = self.pyload.requestFactory.getRequest(self.__name__, self.user)
self.chunk_limit = -1 #: -1 for unlimited
self.resume_download = True
self.premium = self.account.is_premium(self.user)
else:
self.req = self.pyload.requestFactory.getRequest(self.__name__)
self.chunk_limit = 1
self.resume_download = False
self.premium = False
def load_account(self):
if self.req:
self.req.close()
if not self.account:
self.account = self.pyload.accountManager.getAccountPlugin(self.__name__)
if self.account:
if not self.user:
self.user = self.account.select()[0]
if not self.user or not self.account.is_logged(self.user, True):
self.account = False
def preprocessing(self, thread):
"""
Handles important things to do before starting
"""
self.thread = thread
if self.retry_free:
self.account = False
else:
self.load_account() #@TODO: Move to PluginThread in 0.4.10
self.retry_free = False
self._setup()
self.setup()
self.pyload.hookManager.downloadPreparing(self.pyfile) #@TODO: Recheck in 0.4.10
if self.pyfile.abort:
self.abort()
self.pyfile.setStatus("starting")
self.log_debug("PROCESS URL " + self.pyfile.url, "PLUGIN VERSION %s" % self.__version__)
return self.process(self.pyfile)
def process(self, pyfile):
"""
The 'main' method of every plugin, you **have to** overwrite it
"""
raise NotImplementedError
def set_reconnect(self, reconnect):
reconnect = bool(reconnect)
self.log_info(_("RECONNECT ") + ("enabled" if reconnect else "disabled"))
self.log_debug("Previous wantReconnect: %s" % self.wantReconnect)
self.wantReconnect = reconnect
def set_wait(self, seconds, reconnect=None):
"""
Set a specific wait time later used with `wait`
:param seconds: wait time in seconds
:param reconnect: True if a reconnect would avoid wait time
"""
wait_time = max(int(seconds), 1)
wait_until = time.time() + wait_time + 1
self.log_info(_("WAIT %d seconds") % wait_time)
self.log_debug("Previous waitUntil: %f" % self.pyfile.waitUntil)
self.pyfile.waitUntil = wait_until
if reconnect is not None:
self.set_reconnect(reconnect)
def wait(self, seconds=None, reconnect=None):
"""
Waits the time previously set
"""
pyfile = self.pyfile
if seconds is not None:
self.set_wait(seconds)
if reconnect is not None:
self.set_reconnect(reconnect)
self.waiting = True
status = pyfile.status #@NOTE: Remove in 0.4.10
pyfile.setStatus("waiting")
if not self.wantReconnect or self.account:
if self.account:
self.log_warning("Ignore reconnection due logged account")
while pyfile.waitUntil > time.time():
if pyfile.abort:
self.abort()
time.sleep(2)
else:
while pyfile.waitUntil > time.time():
if pyfile.abort:
self.abort()
if self.thread.m.reconnecting.isSet():
self.waiting = False
self.wantReconnect = False
raise Reconnect
self.thread.m.reconnecting.wait(2)
time.sleep(2)
self.waiting = False
pyfile.status = status #@NOTE: Remove in 0.4.10
def skip(self, reason=""):
"""
Skip and give reason
"""
raise Skip(encode(reason)) #@TODO: Remove `encode` in 0.4.10
def abort(self, reason=""):
"""
Abort and give reason
"""
#@TODO: Remove in 0.4.10
if reason:
self.pyfile.error = encode(reason)
raise Abort
def offline(self, reason=""):
"""
Fail and indicate file is offline
"""
#@TODO: Remove in 0.4.10
if reason:
self.pyfile.error = encode(reason)
raise Fail("offline")
def temp_offline(self, reason=""):
"""
Fail and indicates file ist temporary offline, the core may take consequences
"""
#@TODO: Remove in 0.4.10
if reason:
self.pyfile.error = encode(reason)
raise Fail("temp. offline")
def retry(self, max_tries=5, wait_time=1, reason=""):
"""
Retries and begin again from the beginning
:param max_tries: number of maximum retries
:param wait_time: time to wait in seconds
:param reason: reason for retrying, will be passed to fail if max_tries reached
"""
id = inspect.currentframe().f_back.f_lineno
if id not in self.retries:
self.retries[id] = 0
if 0 < max_tries <= self.retries[id]:
self.fail(reason or _("Max retries reached"))
self.wait(wait_time, False)
self.retries[id] += 1
raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10
def restart(self, reason=None, nopremium=False):
if not reason:
reason = _("Fallback to free download") if nopremium else _("Restart")
if nopremium:
if self.premium:
self.retry_free = True
else:
self.fail(reason, _("Download was already free"))
raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10
def fixurl(self, url):
url = _fixurl(url)
if not urlparse.urlparse(url).scheme:
url_p = urlparse.urlparse(self.pyfile.url)
baseurl = "%s://%s" % (url_p.scheme, url_p.netloc)
url = urlparse.urljoin(baseurl, url)
return url
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True):
"""
Downloads the content at url to download folder
:param url:
:param get:
:param post:
:param ref:
:param cookies:
:param disposition: if True and server provides content-disposition header\
the filename will be changed if needed
:return: The location where the file was saved
"""
if self.pyfile.abort:
self.abort()
url = self.fixurl(url)
if not url or not isinstance(url, basestring):
self.fail(_("No url given"))
if self.pyload.debug:
self.log_debug("DOWNLOAD URL " + url,
*["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")])
name = _fixurl(self.pyfile.name)
self.pyfile.name = urlparse.urlparse(name).path.split('/')[-1] or name
self.captcha.correct()
self.check_for_same_files()
self.pyfile.setStatus("downloading")
download_folder = self.pyload.config.get("general", "download_folder")
download_location = fs_join(download_folder, self.pyfile.package().folder)
if not exists(download_location):
try:
os.makedirs(download_location)
except Exception, e:
self.fail(e)
self.set_permissions(download_location)
location = fs_decode(download_location)
filename = os.path.join(location, safe_filename(self.pyfile.name)) #@TODO: Move `safe_filename` check to HTTPDownload in 0.4.10
self.pyload.hookManager.dispatchEvent("download_start", self.pyfile, url, filename)
if self.pyfile.abort:
self.abort()
try:
newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies,
chunks=self.get_chunk_count(), resume=self.resume_download,
progressNotify=self.pyfile.setProgress, disposition=disposition)
finally:
self.pyfile.size = self.req.size
#@TODO: Recheck in 0.4.10
if disposition and newname:
finalname = urlparse.urlparse(newname).path.split('/')[-1].split(' filename*=')[0]
if finalname != newname != self.pyfile.name:
try:
os.rename(fs_join(location, newname), fs_join(location, finalname))
except OSError, e:
self.log_warning(_("Error renaming `%s` to `%s`") % (newname, finalname), e)
finalname = newname
self.log_info(_("`%s` saved as `%s`") % (self.pyfile.name, finalname))
self.pyfile.name = finalname
filename = os.path.join(location, finalname)
class RelinkUs(Crypter):
__name__ = "RelinkUs"
__type__ = "crypter"
__version__ = "3.20"
__status__ = "testing"
__pattern__ = r'http://(?:www\.)?relink\.(?:us|to)/(f/|((view|go)\.php\?id=))(?P<ID>.+)'
__config__ = [("activated" , "bool" , "Activated" , True ),
("use_premium" , "bool" , "Use premium account if available", True ),
("folder_per_package", "Default;Yes;No", "Create folder for each package" , "Default")]
__description__ = """Relink.us decrypter plugin"""
__license__ = "GPLv3"
__authors__ = [("fragonib", "fragonib[AT]yahoo[DOT]es"),
("AndroKev", "*****@*****.**")]
URL_REPLACEMENTS = [(__pattern__ + '.*', r'http://relink.to/f/\g<ID>')]
PREFERRED_LINK_SOURCES = ["cnl2", "dlc", "web"]
OFFLINE_TOKEN = r'<title>Tattooside'
PASSWORD_TOKEN = r'container_password.php'
PASSWORD_ERROR_ROKEN = r'You have entered an incorrect password'
PASSWORD_SUBMIT_URL = r'http://relink.to/container_password.php'
CAPTCHA_TOKEN = r'container_captcha.php'
CIRCLE_CAPTCHA_PATTERN = r'id="captcha_id" value="(\w+?)"'
CAPTCHA_ERROR_ROKEN = r'You have solved the captcha wrong'
CIRCLE_CAPTCHA_IMG_URL = r'http://relink.to/core/captcha/circlecaptcha.php'
CAPTCHA_SUBMIT_URL = r'http://relink.to/container_captcha.php'
FILE_TITLE_PATTERN = r'<th>Title</th><td>(.*)</td></tr>'
FILE_NOTITLE = r'No title'
CNL2_FORM_PATTERN = r'<form id="cnl_form-(.*?)</form>'
CNL2_FORMINPUT_PATTERN = r'<input.*?name="%s".*?value="(.*?)"'
CNL2_JK_KEY = "jk"
CNL2_CRYPTED_KEY = "crypted"
DLC_LINK_PATTERN = r'<a href=".*?" class="dlc_button" target="_blank">'
DLC_DOWNLOAD_URL = r'http://relink.to/download.php'
WEB_FORWARD_PATTERN = r'getFile\(\'(.+)\'\)'
WEB_FORWARD_URL = r'http://relink.to/frame.php'
WEB_LINK_PATTERN = r'<iframe name="Container" height="100%" frameborder="no" width="100%" src="(.+)"></iframe>'
def setup(self):
self.file_id = None
self.package = None
self.captcha = Captcha(self.pyfile)
def decrypt(self, pyfile):
#: Init
self.init_package(pyfile)
#: Request package
self.request_package()
#: Check for online
if not self.is_online():
self.offline()
#: Check for protection
if self.is_password_protected():
self.unlock_password_protection()
self.handle_errors()
if self.is_captcha_protected():
self.unlock_captcha_protection()
self.handle_errors()
#: Get package name and folder
pack_name, folder_name = self.get_package_info()
#: Extract package links
pack_links = []
for sources in self.PREFERRED_LINK_SOURCES:
pack_links.extend(self.handle_link_source(sources))
if pack_links: #: Use only first source which provides links
break
pack_links = set(pack_links)
#: Pack
if pack_links:
self.packages = [(pack_name, pack_links, folder_name)]
def init_package(self, pyfile):
pyfile.url = replace_patterns(pyfile.url, self.URL_REPLACEMENTS)
self.file_id = re.match(self.__pattern__, pyfile.url).group('ID')
self.package = pyfile.package()
def request_package(self):
self.data = self.load(self.pyfile.url)
def is_online(self):
if self.OFFLINE_TOKEN in self.data:
self.log_debug("File not found")
return False
return True
def is_password_protected(self):
if self.PASSWORD_TOKEN in self.data:
self.log_debug("Links are password protected")
return True
def is_captcha_protected(self):
if self.CAPTCHA_TOKEN in self.data:
self.log_debug("Links are captcha protected")
return True
return False
def unlock_password_protection(self):
password = self.get_password()
self.log_debug("Submitting password [%s] for protected links" % password)
if password:
passwd_url = self.PASSWORD_SUBMIT_URL + "?id=%s" % self.file_id
passwd_data = {'id': self.file_id, 'password': password, 'pw': 'submit'}
self.data = self.load(passwd_url, post=passwd_data)
def unlock_captcha_protection(self):
m = re.search(self.CIRCLE_CAPTCHA_PATTERN, self.data)
if m:
self.log_debug("Request circle captcha resolving")
captcha_id = m.group(1)
coords = self.captcha.decrypt(self.CIRCLE_CAPTCHA_IMG_URL, get={'id': captcha_id}, input_type="png", output_type='positional') # , ocr="CircleCaptcha")
self.log_debug("Captcha resolved, coords (%s,%s)" % (coords[0], coords[1]))
post_data = {'button.x' : coords[0],
'button.y' : coords[1],
'captcha_id' : captcha_id,
'captcha_type': "RelinkCircle",
'captcha' : "submit"}
else:
solvemedia = SolveMedia(self.pyfile)
captcha_key = solvemedia.detect_key()
if captcha_key:
self.log_debug(_("Request SolveMedia captcha resolving"))
response, challenge = solvemedia.challenge()
post_data = {'adcopy_response' : response,
'adcopy_challenge': challenge,
'captcha_type' : "Solvemedia",
'submit' : "Continue",
'captcha' : "submit"}
else:
self.log_error(_("Unknown captcha type detected"))
self.fail(_("Unknown captcha type"))
self.data = self.load(self.CAPTCHA_SUBMIT_URL,
ref=False,# ref=self.CAPTCHA_SUBMIT_URL + "&id=" + self.file_id,
get={'id': self.file_id},
post=post_data)
def get_package_info(self):
name = folder = None
#: Try to get info from web
# self.data = self.load(self.pyfile.url)
m = re.search(self.FILE_TITLE_PATTERN, self.data)
if m is not None:
title = m.group(1).strip()
if not self.FILE_NOTITLE in title:
name = folder = title
self.log_debug(_("Found name [%s] and folder [%s] in package info") % (name, folder))
#: Fallback to defaults
if not name or not folder:
name = self.package.name
folder = self.package.folder
self.log_debug(_("Package info not found, defaulting to pyfile name [%s] and folder [%s]") % (name, folder))
#: Return package info
return name, folder
def handle_errors(self):
if self.PASSWORD_ERROR_ROKEN in self.data:
self.fail(_("Wrong password"))
if self.captcha.task:
if self.CAPTCHA_ERROR_ROKEN in self.data:
self.retry_captcha()
else:
self.captcha.correct()
def handle_link_source(self, source):
if source == "cnl2":
return self.handle_CNL2Links()
elif source == "dlc":
return self.handle_DLC_links()
elif source == "web":
return self.handle_WEB_links()
else:
self.error(_('Unknown source type "%s"') % source)
def handle_CNL2Links(self):
self.log_debug(_("Search for CNL2 links"))
pack_links = []
m = re.search(self.CNL2_FORM_PATTERN, self.data, re.S)
if m is not None:
cnl2_form = m.group(1)
try:
(vcrypted, vjk) = self._get_cipher_params(cnl2_form)
for (crypted, jk) in zip(vcrypted, vjk):
pack_links.extend(self._get_links(crypted, jk))
except Exception:
self.log_debug(_("Unable to decrypt CNL2 links", trace=True))
return pack_links
def handle_DLC_links(self):
self.log_debug(_("Search for DLC links"))
pack_links = []
m = re.search(self.DLC_LINK_PATTERN, self.data)
if m is not None:
container_url = self.DLC_DOWNLOAD_URL + "?id=%s&dlc=1" % self.file_id
self.log_debug(_("Downloading DLC container link [%s]") % container_url)
try:
dlc = self.load(container_url)
dlc_filename = self.file_id + ".dlc"
dlc_filepath = fsjoin(self.pyload.config.get('general', 'download_folder'), dlc_filename)
with open(dlc_filepath, "wb") as f:
f.write(dlc)
pack_links.append(dlc_filepath)
except Exception:
self.fail(_("Unable to download DLC container"))
return pack_links
def handle_WEB_links(self):
self.log_debug(_("Search for WEB links"))
pack_links = []
params = re.findall(self.WEB_FORWARD_PATTERN, self.data)
self.log_debug(_("Decrypting %d Web links") % len(params))
for index, param in enumerate(params):
try:
url = self.WEB_FORWARD_URL + "?%s" % param
self.log_debug(_("Decrypting Web link %d, %s") % (index + 1, url))
res = self.load(url)
link = re.search(self.WEB_LINK_PATTERN, res).group(1)
pack_links.append(link)
except Exception, detail:
self.log_debug(_("Error decrypting Web link %s, %s") % (index, detail))
self.wait(4)
return pack_links
