def __init__(self, pyfile):
self._init(pyfile.m.core)
#: Engage wan reconnection
self.wantReconnect = False #@TODO: Change to `want_reconnect` in 0.4.10
#: Enable simultaneous processing of multiple downloads
self.multiDL = True #@TODO: Change to `multi_dl` in 0.4.10
self.limitDL = 0 #@TODO: Change to `limit_dl` in 0.4.10
#: time.time() + wait in seconds
self.wait_until = 0
self.waiting = False
#: Account handler instance, see :py:class:`Account`
self.account = None
self.user = None
self.req = None #: Browser instance, see `network.Browser`
#: Associated pyfile instance, see `PyFile`
self.pyfile = pyfile
self.thread = None #: Holds thread in future
#: Location where the last call to download was saved
self.last_download = ""
#: Re match of the last call to `checkDownload`
self.last_check = None
#: Js engine, see `JsEngine`
self.js = self.pyload.js
#: Captcha stuff
self.captcha = Captcha(self)
#: Some plugins store html code here
self.html = None
#: Dict of the amount of retries already made
self.retries = {}
self.retry_free = False #@TODO: Recheck in 0.4.10
self._setup()
self.init()
def __init__(self, pyfile):
self._init(pyfile.m.core)
#: Engage wan reconnection
self.wantReconnect = False #@TODO: Change to `want_reconnect` in 0.4.10
#: Enable simultaneous processing of multiple downloads
self.multiDL = True #@TODO: Change to `multi_dl` in 0.4.10
#: time.time() + wait in seconds
self.waiting = False
#: Account handler instance, see :py:class:`Account`
self.account = None
self.user = None #@TODO: Remove in 0.4.10
self.premium = None
#: Associated pyfile instance, see `PyFile`
self.pyfile = pyfile
#: Holds thread in future
self.thread = None
#: Js engine, see `JsEngine`
self.js = self.pyload.js
#: Captcha stuff
#@TODO: Replace in 0.4.10:
#_Captcha = self.pyload.pluginManager.loadClass("captcha", self.classname) or Captcha
# self.captcha = _Captcha(pyfile)
self.captcha = Captcha(pyfile)
#: Some plugins store html code here
self.data = ""
#: Dict of the amount of retries already made
self.retries = {}
self.init_base()
self.init()
class Hoster(Plugin):
__name__ = "Hoster"
__type__ = "hoster"
__version__ = "0.18"
__status__ = "testing"
__pattern__ = r'^unmatchable$'
__config__ = [] #: [("name", "type", "desc", "default")]
__description__ = """Base hoster plugin"""
__license__ = "GPLv3"
__authors__ = [("RaNaN" , "*****@*****.**" ),
("spoob" , "*****@*****.**" ),
("mkaay" , "*****@*****.**" ),
("Walter Purcaro", "*****@*****.**")]
def __init__(self, pyfile):
self._init(pyfile.m.core)
#: Engage wan reconnection
self.wantReconnect = False #@TODO: Change to `want_reconnect` in 0.4.10
#: Enable simultaneous processing of multiple downloads
self.multiDL = True #@TODO: Change to `multi_dl` in 0.4.10
self.limitDL = 0 #@TODO: Change to `limit_dl` in 0.4.10
#: time.time() + wait in seconds
self.wait_until = 0
self.waiting = False
#: Account handler instance, see :py:class:`Account`
self.account = None
self.user = None
self.req = None #: Browser instance, see `network.Browser`
#: Associated pyfile instance, see `PyFile`
self.pyfile = pyfile
self.thread = None #: Holds thread in future
#: Location where the last call to download was saved
self.last_download = ""
#: Re match of the last call to `checkDownload`
self.last_check = None
#: Js engine, see `JsEngine`
self.js = self.pyload.js
#: Captcha stuff
self.captcha = Captcha(self)
#: Some plugins store html code here
self.html = None
#: Dict of the amount of retries already made
self.retries = {}
self.retry_free = False #@TODO: Recheck in 0.4.10
self._setup()
self.init()
@classmethod
def get_info(cls, url="", html=""):
url = _fixurl(url)
url_p = urlparse.urlparse(url)
return {'name' : (url_p.path.split('/')[-1] or
url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] or
url_p.netloc.split('.', 1)[0]),
'size' : 0,
'status': 3 if url else 8,
'url' : url}
def init(self):
"""
Initialize the plugin (in addition to `__init__`)
"""
pass
def setup(self):
"""
Setup for enviroment and other things, called before downloading (possibly more than one time)
"""
pass
def _setup(self):
if self.account:
self.req = self.pyload.requestFactory.getRequest(self.__name__, self.user)
self.chunk_limit = -1 #: -1 for unlimited
self.resume_download = True
self.premium = self.account.is_premium(self.user)
else:
self.req = self.pyload.requestFactory.getRequest(self.__name__)
self.chunk_limit = 1
self.resume_download = False
self.premium = False
def load_account(self):
if self.req:
self.req.close()
if not self.account:
self.account = self.pyload.accountManager.getAccountPlugin(self.__name__)
if self.account:
if not self.user:
self.user = self.account.select()[0]
if not self.user or not self.account.is_logged(self.user, True):
self.account = False
def preprocessing(self, thread):
"""
Handles important things to do before starting
"""
self.thread = thread
if self.retry_free:
self.account = False
else:
self.load_account() #@TODO: Move to PluginThread in 0.4.10
self.retry_free = False
self._setup()
self.setup()
self.pyload.hookManager.downloadPreparing(self.pyfile) #@TODO: Recheck in 0.4.10
if self.pyfile.abort:
self.abort()
self.pyfile.setStatus("starting")
self.log_debug("PROCESS URL " + self.pyfile.url, "PLUGIN VERSION %s" % self.__version__)
return self.process(self.pyfile)
def process(self, pyfile):
"""
The 'main' method of every plugin, you **have to** overwrite it
"""
raise NotImplementedError
def set_reconnect(self, reconnect):
reconnect = bool(reconnect)
self.log_info(_("RECONNECT ") + ("enabled" if reconnect else "disabled"))
self.log_debug("Previous wantReconnect: %s" % self.wantReconnect)
self.wantReconnect = reconnect
def set_wait(self, seconds, reconnect=None):
"""
Set a specific wait time later used with `wait`
:param seconds: wait time in seconds
:param reconnect: True if a reconnect would avoid wait time
"""
wait_time = max(int(seconds), 1)
wait_until = time.time() + wait_time + 1
self.log_info(_("WAIT %d seconds") % wait_time)
self.log_debug("Previous waitUntil: %f" % self.pyfile.waitUntil)
self.pyfile.waitUntil = wait_until
if reconnect is not None:
self.set_reconnect(reconnect)
def wait(self, seconds=None, reconnect=None):
"""
Waits the time previously set
"""
pyfile = self.pyfile
if seconds is not None:
self.set_wait(seconds)
if reconnect is not None:
self.set_reconnect(reconnect)
self.waiting = True
status = pyfile.status #@NOTE: Remove in 0.4.10
pyfile.setStatus("waiting")
if not self.wantReconnect or self.account:
if self.account:
self.log_warning("Ignore reconnection due logged account")
while pyfile.waitUntil > time.time():
if pyfile.abort:
self.abort()
time.sleep(2)
else:
while pyfile.waitUntil > time.time():
if pyfile.abort:
self.abort()
if self.thread.m.reconnecting.isSet():
self.waiting = False
self.wantReconnect = False
raise Reconnect
self.thread.m.reconnecting.wait(2)
time.sleep(2)
self.waiting = False
pyfile.status = status #@NOTE: Remove in 0.4.10
def skip(self, reason=""):
"""
Skip and give reason
"""
raise Skip(encode(reason)) #@TODO: Remove `encode` in 0.4.10
def abort(self, reason=""):
"""
Abort and give reason
"""
#@TODO: Remove in 0.4.10
if reason:
self.pyfile.error = encode(reason)
raise Abort
def offline(self, reason=""):
"""
Fail and indicate file is offline
"""
#@TODO: Remove in 0.4.10
if reason:
self.pyfile.error = encode(reason)
raise Fail("offline")
def temp_offline(self, reason=""):
"""
Fail and indicates file ist temporary offline, the core may take consequences
"""
#@TODO: Remove in 0.4.10
if reason:
self.pyfile.error = encode(reason)
raise Fail("temp. offline")
def retry(self, max_tries=5, wait_time=1, reason=""):
"""
Retries and begin again from the beginning
:param max_tries: number of maximum retries
:param wait_time: time to wait in seconds
:param reason: reason for retrying, will be passed to fail if max_tries reached
"""
id = inspect.currentframe().f_back.f_lineno
if id not in self.retries:
self.retries[id] = 0
if 0 < max_tries <= self.retries[id]:
self.fail(reason or _("Max retries reached"))
self.wait(wait_time, False)
self.retries[id] += 1
raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10
def restart(self, reason=None, nopremium=False):
if not reason:
reason = _("Fallback to free download") if nopremium else _("Restart")
if nopremium:
if self.premium:
self.retry_free = True
else:
self.fail(reason, _("Download was already free"))
raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10
def fixurl(self, url):
url = _fixurl(url)
if not urlparse.urlparse(url).scheme:
url_p = urlparse.urlparse(self.pyfile.url)
baseurl = "%s://%s" % (url_p.scheme, url_p.netloc)
url = urlparse.urljoin(baseurl, url)
return url
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True):
"""
Downloads the content at url to download folder
:param url:
:param get:
:param post:
:param ref:
:param cookies:
:param disposition: if True and server provides content-disposition header\
the filename will be changed if needed
:return: The location where the file was saved
"""
if self.pyfile.abort:
self.abort()
url = self.fixurl(url)
if not url or not isinstance(url, basestring):
self.fail(_("No url given"))
if self.pyload.debug:
self.log_debug("DOWNLOAD URL " + url,
*["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")])
name = _fixurl(self.pyfile.name)
self.pyfile.name = urlparse.urlparse(name).path.split('/')[-1] or name
self.captcha.correct()
self.check_for_same_files()
self.pyfile.setStatus("downloading")
download_folder = self.pyload.config.get("general", "download_folder")
download_location = fs_join(download_folder, self.pyfile.package().folder)
if not exists(download_location):
try:
os.makedirs(download_location)
except Exception, e:
self.fail(e)
self.set_permissions(download_location)
location = fs_decode(download_location)
filename = os.path.join(location, safe_filename(self.pyfile.name)) #@TODO: Move `safe_filename` check to HTTPDownload in 0.4.10
self.pyload.hookManager.dispatchEvent("download_start", self.pyfile, url, filename)
if self.pyfile.abort:
self.abort()
try:
newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies,
chunks=self.get_chunk_count(), resume=self.resume_download,
progressNotify=self.pyfile.setProgress, disposition=disposition)
finally:
self.pyfile.size = self.req.size
#@TODO: Recheck in 0.4.10
if disposition and newname:
finalname = urlparse.urlparse(newname).path.split('/')[-1].split(' filename*=')[0]
if finalname != newname != self.pyfile.name:
try:
os.rename(fs_join(location, newname), fs_join(location, finalname))
except OSError, e:
self.log_warning(_("Error renaming `%s` to `%s`") % (newname, finalname), e)
finalname = newname
self.log_info(_("`%s` saved as `%s`") % (self.pyfile.name, finalname))
self.pyfile.name = finalname
filename = os.path.join(location, finalname)
def setup(self):
self.file_id = None
self.package = None
self.captcha = Captcha(self.pyfile)
class RelinkUs(Crypter):
__name__ = "RelinkUs"
__type__ = "crypter"
__version__ = "3.20"
__status__ = "testing"
__pattern__ = r'http://(?:www\.)?relink\.(?:us|to)/(f/|((view|go)\.php\?id=))(?P<ID>.+)'
__config__ = [("activated" , "bool" , "Activated" , True ),
("use_premium" , "bool" , "Use premium account if available", True ),
("folder_per_package", "Default;Yes;No", "Create folder for each package" , "Default")]
__description__ = """Relink.us decrypter plugin"""
__license__ = "GPLv3"
__authors__ = [("fragonib", "fragonib[AT]yahoo[DOT]es"),
("AndroKev", "*****@*****.**")]
URL_REPLACEMENTS = [(__pattern__ + '.*', r'http://relink.to/f/\g<ID>')]
PREFERRED_LINK_SOURCES = ["cnl2", "dlc", "web"]
OFFLINE_TOKEN = r'<title>Tattooside'
PASSWORD_TOKEN = r'container_password.php'
PASSWORD_ERROR_ROKEN = r'You have entered an incorrect password'
PASSWORD_SUBMIT_URL = r'http://relink.to/container_password.php'
CAPTCHA_TOKEN = r'container_captcha.php'
CIRCLE_CAPTCHA_PATTERN = r'id="captcha_id" value="(\w+?)"'
CAPTCHA_ERROR_ROKEN = r'You have solved the captcha wrong'
CIRCLE_CAPTCHA_IMG_URL = r'http://relink.to/core/captcha/circlecaptcha.php'
CAPTCHA_SUBMIT_URL = r'http://relink.to/container_captcha.php'
FILE_TITLE_PATTERN = r'<th>Title</th><td>(.*)</td></tr>'
FILE_NOTITLE = r'No title'
CNL2_FORM_PATTERN = r'<form id="cnl_form-(.*?)</form>'
CNL2_FORMINPUT_PATTERN = r'<input.*?name="%s".*?value="(.*?)"'
CNL2_JK_KEY = "jk"
CNL2_CRYPTED_KEY = "crypted"
DLC_LINK_PATTERN = r'<a href=".*?" class="dlc_button" target="_blank">'
DLC_DOWNLOAD_URL = r'http://relink.to/download.php'
WEB_FORWARD_PATTERN = r'getFile\(\'(.+)\'\)'
WEB_FORWARD_URL = r'http://relink.to/frame.php'
WEB_LINK_PATTERN = r'<iframe name="Container" height="100%" frameborder="no" width="100%" src="(.+)"></iframe>'
def setup(self):
self.file_id = None
self.package = None
self.captcha = Captcha(self.pyfile)
def decrypt(self, pyfile):
#: Init
self.init_package(pyfile)
#: Request package
self.request_package()
#: Check for online
if not self.is_online():
self.offline()
#: Check for protection
if self.is_password_protected():
self.unlock_password_protection()
self.handle_errors()
if self.is_captcha_protected():
self.unlock_captcha_protection()
self.handle_errors()
#: Get package name and folder
pack_name, folder_name = self.get_package_info()
#: Extract package links
pack_links = []
for sources in self.PREFERRED_LINK_SOURCES:
pack_links.extend(self.handle_link_source(sources))
if pack_links: #: Use only first source which provides links
break
pack_links = set(pack_links)
#: Pack
if pack_links:
self.packages = [(pack_name, pack_links, folder_name)]
def init_package(self, pyfile):
pyfile.url = replace_patterns(pyfile.url, self.URL_REPLACEMENTS)
self.file_id = re.match(self.__pattern__, pyfile.url).group('ID')
self.package = pyfile.package()
def request_package(self):
self.data = self.load(self.pyfile.url)
def is_online(self):
if self.OFFLINE_TOKEN in self.data:
self.log_debug("File not found")
return False
return True
def is_password_protected(self):
if self.PASSWORD_TOKEN in self.data:
self.log_debug("Links are password protected")
return True
def is_captcha_protected(self):
if self.CAPTCHA_TOKEN in self.data:
self.log_debug("Links are captcha protected")
return True
return False
def unlock_password_protection(self):
password = self.get_password()
self.log_debug("Submitting password [%s] for protected links" % password)
if password:
passwd_url = self.PASSWORD_SUBMIT_URL + "?id=%s" % self.file_id
passwd_data = {'id': self.file_id, 'password': password, 'pw': 'submit'}
self.data = self.load(passwd_url, post=passwd_data)
def unlock_captcha_protection(self):
m = re.search(self.CIRCLE_CAPTCHA_PATTERN, self.data)
if m:
self.log_debug("Request circle captcha resolving")
captcha_id = m.group(1)
coords = self.captcha.decrypt(self.CIRCLE_CAPTCHA_IMG_URL, get={'id': captcha_id}, input_type="png", output_type='positional') # , ocr="CircleCaptcha")
self.log_debug("Captcha resolved, coords (%s,%s)" % (coords[0], coords[1]))
post_data = {'button.x' : coords[0],
'button.y' : coords[1],
'captcha_id' : captcha_id,
'captcha_type': "RelinkCircle",
'captcha' : "submit"}
else:
solvemedia = SolveMedia(self.pyfile)
captcha_key = solvemedia.detect_key()
if captcha_key:
self.log_debug(_("Request SolveMedia captcha resolving"))
response, challenge = solvemedia.challenge()
post_data = {'adcopy_response' : response,
'adcopy_challenge': challenge,
'captcha_type' : "Solvemedia",
'submit' : "Continue",
'captcha' : "submit"}
else:
self.log_error(_("Unknown captcha type detected"))
self.fail(_("Unknown captcha type"))
self.data = self.load(self.CAPTCHA_SUBMIT_URL,
ref=False,# ref=self.CAPTCHA_SUBMIT_URL + "&id=" + self.file_id,
get={'id': self.file_id},
post=post_data)
def get_package_info(self):
name = folder = None
#: Try to get info from web
# self.data = self.load(self.pyfile.url)
m = re.search(self.FILE_TITLE_PATTERN, self.data)
if m is not None:
title = m.group(1).strip()
if not self.FILE_NOTITLE in title:
name = folder = title
self.log_debug(_("Found name [%s] and folder [%s] in package info") % (name, folder))
#: Fallback to defaults
if not name or not folder:
name = self.package.name
folder = self.package.folder
self.log_debug(_("Package info not found, defaulting to pyfile name [%s] and folder [%s]") % (name, folder))
#: Return package info
return name, folder
def handle_errors(self):
if self.PASSWORD_ERROR_ROKEN in self.data:
self.fail(_("Wrong password"))
if self.captcha.task:
if self.CAPTCHA_ERROR_ROKEN in self.data:
self.retry_captcha()
else:
self.captcha.correct()
def handle_link_source(self, source):
if source == "cnl2":
return self.handle_CNL2Links()
elif source == "dlc":
return self.handle_DLC_links()
elif source == "web":
return self.handle_WEB_links()
else:
self.error(_('Unknown source type "%s"') % source)
def handle_CNL2Links(self):
self.log_debug(_("Search for CNL2 links"))
pack_links = []
m = re.search(self.CNL2_FORM_PATTERN, self.data, re.S)
if m is not None:
cnl2_form = m.group(1)
try:
(vcrypted, vjk) = self._get_cipher_params(cnl2_form)
for (crypted, jk) in zip(vcrypted, vjk):
pack_links.extend(self._get_links(crypted, jk))
except Exception:
self.log_debug(_("Unable to decrypt CNL2 links", trace=True))
return pack_links
def handle_DLC_links(self):
self.log_debug(_("Search for DLC links"))
pack_links = []
m = re.search(self.DLC_LINK_PATTERN, self.data)
if m is not None:
container_url = self.DLC_DOWNLOAD_URL + "?id=%s&dlc=1" % self.file_id
self.log_debug(_("Downloading DLC container link [%s]") % container_url)
try:
dlc = self.load(container_url)
dlc_filename = self.file_id + ".dlc"
dlc_filepath = fsjoin(self.pyload.config.get('general', 'download_folder'), dlc_filename)
with open(dlc_filepath, "wb") as f:
f.write(dlc)
pack_links.append(dlc_filepath)
except Exception:
self.fail(_("Unable to download DLC container"))
return pack_links
def handle_WEB_links(self):
self.log_debug(_("Search for WEB links"))
pack_links = []
params = re.findall(self.WEB_FORWARD_PATTERN, self.data)
self.log_debug(_("Decrypting %d Web links") % len(params))
for index, param in enumerate(params):
try:
url = self.WEB_FORWARD_URL + "?%s" % param
self.log_debug(_("Decrypting Web link %d, %s") % (index + 1, url))
res = self.load(url)
link = re.search(self.WEB_LINK_PATTERN, res).group(1)
pack_links.append(link)
except Exception, detail:
self.log_debug(_("Error decrypting Web link %s, %s") % (index, detail))
self.wait(4)
return pack_links
class Base(Plugin):
__name__ = "Base"
__type__ = "base"
__version__ = "0.25"
__status__ = "stable"
__pattern__ = r'^unmatchable$'
__config__ = [("activated" , "bool", "Activated" , True),
("use_premium", "bool", "Use premium account if available", True)]
__description__ = """Base plugin for Hoster and Crypter"""
__license__ = "GPLv3"
__authors__ = [("Walter Purcaro", "*****@*****.**")]
URL_REPLACEMENTS = []
@classmethod
def get_info(cls, url="", html=""):
url = fixurl(url, unquote=True)
info = {'name' : parse_name(url),
'hash' : {},
'pattern': {},
'size' : 0,
'status' : 7 if url else 8,
'url' : replace_patterns(url, cls.URL_REPLACEMENTS)}
try:
info['pattern'] = re.match(cls.__pattern__, url).groupdict()
except Exception:
pass
return info
def __init__(self, pyfile):
self._init(pyfile.m.core)
#: Engage wan reconnection
self.wantReconnect = False #@TODO: Change to `want_reconnect` in 0.4.10
#: Enable simultaneous processing of multiple downloads
self.multiDL = True #@TODO: Change to `multi_dl` in 0.4.10
#: time.time() + wait in seconds
self.waiting = False
#: Account handler instance, see :py:class:`Account`
self.account = None
self.user = None #@TODO: Remove in 0.4.10
self.premium = None
#: Associated pyfile instance, see `PyFile`
self.pyfile = pyfile
#: Holds thread in future
self.thread = None
#: Js engine, see `JsEngine`
self.js = self.pyload.js
#: Captcha stuff
#@TODO: Replace in 0.4.10:
#_Captcha = self.pyload.pluginManager.loadClass("captcha", self.classname) or Captcha
# self.captcha = _Captcha(pyfile)
self.captcha = Captcha(pyfile)
#: Some plugins store html code here
self.data = ""
#: Dict of the amount of retries already made
self.retries = {}
self.init_base()
self.init()
def _log(self, level, plugintype, pluginname, messages):
log = getattr(self.pyload.log, level)
msg = u" | ".join(decode(a).strip() for a in messages if a)
#: Hide any password
try:
msg = msg.replace(self.account.info['login']['password'], "**********")
except Exception:
pass
log("%(plugintype)s %(pluginname)s[%(id)s]: %(msg)s" %
{'plugintype': plugintype.upper(),
'pluginname': pluginname,
'id' : self.pyfile.id,
'msg' : msg})
def init_base(self):
pass
def setup_base(self):
pass
def setup(self):
"""
Setup for enviroment and other things, called before downloading (possibly more than one time)
"""
pass
def _setup(self):
#@TODO: Remove in 0.4.10
self.pyfile.error = ""
self.data = ""
self.last_html = ""
self.last_header = {}
if self.config.get('use_premium', True):
self.load_account() #@TODO: Move to PluginThread in 0.4.10
else:
self.account = False
self.user = None #@TODO: Remove in 0.4.10
try:
self.req.close()
except Exception:
pass
if self.account:
self.req = self.pyload.requestFactory.getRequest(self.classname, self.account.user)
self.premium = self.account.info['data']['premium'] #@NOTE: Avoid one unnecessary get_info call by `self.account.premium` here
else:
self.req = self.pyload.requestFactory.getRequest(self.classname)
self.premium = False
self.req.setOption("timeout", 60) #@TODO: Remove in 0.4.10
self.setup_base()
self.grab_info()
self.setup()
def load_account(self):
if not self.account:
self.account = self.pyload.accountManager.getAccountPlugin(self.classname)
if not self.account:
self.account = False
self.user = None #@TODO: Remove in 0.4.10
else:
self.account.choose()
self.user = self.account.user #@TODO: Remove in 0.4.10
if self.account.user is None:
self.account = False
def _update_name(self):
name = self.info.get('name')
if name and name != self.info.get('url'):
self.pyfile.name = name
else:
name = self.pyfile.name
self.log_info(_("Link name: ") + name)
def _update_size(self):
size = self.info.get('size')
if size > 0:
self.pyfile.size = int(self.info.get('size')) #@TODO: Fix int conversion in 0.4.10
else:
size = self.pyfile.size
if size:
self.log_info(_("Link size: %s (%s bytes)") % (format_size(size), size))
else:
self.log_info(_("Link size: N/D"))
def _update_status(self):
self.pyfile.status = self.info.get('status', 14)
self.pyfile.sync()
self.log_info(_("Link status: ") + self.pyfile.getStatusName())
def sync_info(self):
self._update_name()
self._update_size()
self._update_status()
def grab_info(self):
self.log_info(_("Grabbing link info..."))
old_info = dict(self.info)
new_info = self.get_info(self.pyfile.url, self.data)
self.info.update(new_info)
self.log_debug("Link info: %s" % self.info)
self.log_debug("Previous link info: %s" % old_info)
self.sync_info()
def check_status(self):
status = self.pyfile.status
if status == 1:
self.offline()
elif status == 4:
self.skip(self.pyfile.statusname)
elif status == 6:
self.temp_offline()
elif status == 8:
self.fail()
elif status == 9 or self.pyfile.abort:
self.abort()
def _initialize(self):
self.log_debug("Plugin version: " + self.__version__)
self.log_debug("Plugin status: " + self.__status__)
if self.__status__ == "broken":
self.abort(_("Plugin is temporarily unavailable"))
elif self.__status__ == "testing":
self.log_warning(_("Plugin may be unstable"))
def _process(self, thread):
"""
Handles important things to do before starting
"""
self.thread = thread
self._initialize()
self._setup()
#@TODO: Enable in 0.4.10
# self.pyload.hookManager.downloadPreparing(self.pyfile)
# self.check_status()
self.pyfile.setStatus("starting")
self.log_info(_("Processing url: ") + self.pyfile.url)
self.process(self.pyfile)
self.check_status()
#: Deprecated method, use `_process` instead (Remove in 0.4.10)
def preprocessing(self, *args, **kwargs):
#@NOTE: Set pyfile status from `queued` to `starting` as soon as possible to avoid race condition in ThreadManager's assignJob function
#@NOTE: Move to ThreadManager in 0.4.10
self.pyfile.setStatus("starting")
#@NOTE: Recheck info thread synchronization in 0.4.10
return self._process(*args, **kwargs)
def process(self, pyfile):
"""
The "main" method of every hoster plugin, you **have to** overwrite it
"""
raise NotImplementedError
def set_reconnect(self, reconnect):
self.log_debug("RECONNECT %s required" % ("" if reconnect else "not"),
"Previous wantReconnect: %s" % self.wantReconnect)
self.wantReconnect = bool(reconnect)
return True
def set_wait(self, seconds, strict=False):
"""
Set a specific wait time later used with `wait`
:param seconds: wait time in seconds
:param reconnect: True if a reconnect would avoid wait time
"""
wait_time = float(seconds)
if wait_time < 0:
return False
old_wait_until = self.pyfile.waitUntil
new_wait_until = time.time() + wait_time + float(not strict)
self.log_debug("WAIT set to timestamp %f" % new_wait_until,
"Previous waitUntil: %f" % old_wait_until)
self.pyfile.waitUntil = new_wait_until
return True
def wait(self, seconds=None, reconnect=None):
"""
Waits the time previously set
"""
if seconds is not None:
self.set_wait(seconds)
if reconnect is not None:
self.set_reconnect(reconnect)
wait_time = self.pyfile.waitUntil - time.time()
if wait_time < 1:
self.log_warning(_("Invalid wait time interval"))
return
self.waiting = True
status = self.pyfile.status #@NOTE: Recheck in 0.4.10
self.pyfile.setStatus("waiting")
self.log_info(_("Waiting %s...") % format_time(wait_time))
if self.wantReconnect:
self.log_info(_("Requiring reconnection..."))
if self.account:
self.log_warning(_("Reconnection ignored due logged account"))
if not self.wantReconnect or self.account:
while self.pyfile.waitUntil > time.time():
self.check_status()
time.sleep(2)
else:
while self.pyfile.waitUntil > time.time():
self.check_status()
self.thread.m.reconnecting.wait(1)
if self.thread.m.reconnecting.isSet():
self.waiting = False
self.wantReconnect = False
self.req.clearCookies()
raise Reconnect
time.sleep(2)
self.waiting = False
self.pyfile.status = status #@NOTE: Recheck in 0.4.10
def skip(self, msg=""):
"""
Skip and give msg
"""
raise Skip(encode(msg or self.pyfile.error or self.pyfile.pluginname)) #@TODO: Remove `encode` in 0.4.10
#@TODO: Remove in 0.4.10
def fail(self, msg=""):
"""
Fail and give msg
"""
msg = msg.strip()
if msg:
self.pyfile.error = msg
else:
msg = self.pyfile.error or self.info.get('error') or self.pyfile.getStatusName()
raise Fail(encode(msg)) #@TODO: Remove `encode` in 0.4.10
def error(self, msg="", type=_("Parse")):
type = _("%s error") % type.strip().capitalize() if type else _("Unknown")
msg = _("%(type)s: %(msg)s | Plugin may be out of date"
% {'type': type, 'msg': msg or self.pyfile.error})
self.fail(msg)
def abort(self, msg=""):
"""
Abort and give msg
"""
if msg: #@TODO: Remove in 0.4.10
self.pyfile.error = encode(msg)
raise Abort
#@TODO: Recheck in 0.4.10
def offline(self, msg=""):
"""
Fail and indicate file is offline
"""
self.fail("offline")
#@TODO: Recheck in 0.4.10
def temp_offline(self, msg=""):
"""
Fail and indicates file ist temporary offline, the core may take consequences
"""
self.fail("temp. offline")
def restart(self, msg="", premium=True):
if not msg:
msg = _("Restart plugin") if premium else _("Fallback to free processing")
if not premium:
if self.premium:
self.restart_free = True
else:
self.fail("%s | %s" % (msg, _("Url was already processed as free")))
self.req.clearCookies()
raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10
def retry(self, attemps=5, wait=1, msg=""):
"""
Retries and begin again from the beginning
:param attemps: number of maximum retries
:param wait: time to wait in seconds before retry
:param msg: message passed to fail if attemps value was reached
"""
frame = inspect.currentframe()
try:
id = frame.f_back.f_lineno
finally:
del frame #: Delete the frame or it wont be cleaned
if id not in self.retries:
self.retries[id] = 0
if 0 < attemps <= self.retries[id]:
self.fail(msg or _("Max retries reached"))
self.wait(wait, False)
self.retries[id] += 1
raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10
def retry_captcha(self, attemps=10, wait=1, msg=_("Max captcha retries reached")):
self.captcha.invalid()
self.retry(attemps, wait, msg)
def fixurl(self, url, baseurl=None, unquote=True):
url = fixurl(url, unquote=True)
baseurl = fixurl(baseurl or self.pyfile.url, unquote=True)
if not urlparse.urlparse(url).scheme:
url_p = urlparse.urlparse(baseurl)
baseurl = "%s://%s" % (url_p.scheme, url_p.netloc)
url = urlparse.urljoin(baseurl, url)
return fixurl(url, unquote)
def load(self, *args, **kwargs):
self.check_status()
return super(Base, self).load(*args, **kwargs)
def parse_html_form(self, attr_str="", input_names={}):
return parse_html_form(attr_str, self.data, input_names)
def get_password(self):
"""
Get the password the user provided in the package
"""
return self.pyfile.package().password or ""
def clean(self):
"""
Clean everything and remove references
"""
super(Base, self).clean()
for attr in ("account", "html", "pyfile", "thread"):
if hasattr(self, attr):
setattr(self, attr, None)
