def downloads():
root = PYLOAD.getConfigValue("general", "download_folder")
if not isdir(root):
return base([_('Download directory not found.')])
data = {
'folder': [],
'files': []
}
items = listdir(fs_encode(root))
for item in sorted([fs_decode(x) for x in items]):
if isdir(save_join(root, item)):
folder = {
'name': item,
'path': item,
'files': []
}
files = listdir(save_join(root, item))
for file in sorted([fs_decode(x) for x in files]):
try:
if isfile(save_join(root, item, file)):
folder['files'].append(file)
except:
pass
data['folder'].append(folder)
elif isfile(join(root, item)):
data['files'].append(item)
return render_to_response('downloads.html', {'files': data}, [pre_processor])
def downloads():
root = PYLOAD.getConfigValue("general", "download_folder")
if not isdir(root):
return base([_('Download directory not found.')])
data = {'folder': [], 'files': []}
items = listdir(fs_encode(root))
for item in sorted([fs_decode(x) for x in items]):
if isdir(save_join(root, item)):
folder = {'name': item, 'path': item, 'files': []}
files = listdir(save_join(root, item))
for file in sorted([fs_decode(x) for x in files]):
try:
if isfile(save_join(root, item, file)):
folder['files'].append(file)
except:
pass
data['folder'].append(folder)
elif isfile(join(root, item)):
data['files'].append(item)
return render_to_response('downloads.html', {'files': data},
[pre_processor])
def list(self, password=None):
command = "vb" if self.fullpath else "lb"
p = self.call_cmd(command, "-v", fs_encode(self.filename), password=password)
out, err = p.communicate()
if "Cannot open" in err:
raise ArchiveError(_("Cannot open file"))
if err.strip(): #: only log error at this point
self.manager.logError(err.strip())
result = set()
if not self.fullpath and self.VERSION.startswith('5'):
# NOTE: Unrar 5 always list full path
for f in fs_decode(out).splitlines():
f = save_join(self.out, os.path.basename(f.strip()))
if os.path.isfile(f):
result.add(save_join(self.out, os.path.basename(f)))
else:
for f in fs_decode(out).splitlines():
f = f.strip()
result.add(save_join(self.out, f))
return list(result)
def downloads():
root = PYLOAD.getConfigValue("general", "download_folder")
if not isdir(root):
return base([_("Download directory not found.")])
data = {"folder": [], "files": []}
items = listdir(fs_encode(root))
for item in sorted([fs_decode(x) for x in items]):
if isdir(safe_join(root, item)):
folder = {"name": item, "path": item, "files": []}
files = listdir(safe_join(root, item))
for file in sorted([fs_decode(x) for x in files]):
try:
if isfile(safe_join(root, item, file)):
folder["files"].append(file)
except:
pass
data["folder"].append(folder)
elif isfile(join(root, item)):
data["files"].append(item)
return render_to_response("downloads.html", {"files": data}, [pre_processor])
def list(self, password=None):
command = "vb" if self.fullpath else "lb"
p = self.call_cmd(command,
"-v",
fs_encode(self.filename),
password=password)
out, err = p.communicate()
if "Cannot open" in err:
raise ArchiveError(_("Cannot open file"))
if err.strip(): #: only log error at this point
self.manager.logError(err.strip())
result = set()
if not self.fullpath and self.VERSION.startswith('5'):
# NOTE: Unrar 5 always list full path
for f in fs_decode(out).splitlines():
f = save_join(self.out, os.path.basename(f.strip()))
if os.path.isfile(f):
result.add(save_join(self.out, os.path.basename(f)))
else:
for f in fs_decode(out).splitlines():
f = f.strip()
result.add(save_join(self.out, f))
return list(result)
class MegaCoNz(Hoster):
__name__ = "MegaCoNz"
__type__ = "hoster"
__version__ = "0.30"
__pattern__ = r'(https?://(?:www\.)?mega(\.co)?\.nz/|mega:|chrome:.+?)#(?P<TYPE>N|)!(?P<ID>[\w^_]+)!(?P<KEY>[\w,-]+)'
__description__ = """Mega.co.nz hoster plugin"""
__license__ = "GPLv3"
__authors__ = [("RaNaN", "*****@*****.**"),
("Walter Purcaro", "*****@*****.**")]
API_URL = "https://eu.api.mega.co.nz/cs"
FILE_SUFFIX = ".crypted"
def b64_decode(self, data):
data = data.replace("-", "+").replace("_", "/")
return standard_b64decode(data + '=' * (-len(data) % 4))
def getCipherKey(self, key):
""" Construct the cipher key from the given data """
a = array.array("I", self.b64_decode(key))
k = array.array("I", (a[0] ^ a[4], a[1] ^ a[5], a[2] ^ a[6], a[3] ^ a[7]))
iv = a[4:6] + array.array("I", (0, 0))
meta_mac = a[6:8]
return k, iv, meta_mac
def api_response(self, **kwargs):
""" Dispatch a call to the api, see https://mega.co.nz/#developers """
# generate a session id, no idea where to obtain elsewhere
uid = random.randint(10 << 9, 10 ** 10)
res = self.load(self.API_URL, get={'id': uid}, post=json_dumps([kwargs]))
self.logDebug("Api Response: " + res)
return json_loads(res)
def decryptAttr(self, data, key):
k, iv, meta_mac = self.getCipherKey(key)
cbc = AES.new(k, AES.MODE_CBC, "\0" * 16)
attr = decode(cbc.decrypt(self.b64_decode(data)))
self.logDebug("Decrypted Attr: %s" % attr)
if not attr.startswith("MEGA"):
self.fail(_("Decryption failed"))
# Data is padded, 0-bytes must be stripped
return json_loads(re.search(r'{.+?}', attr).group(0))
def decryptFile(self, key):
""" Decrypts the file at lastDownload` """
# upper 64 bit of counter start
n = self.b64_decode(key)[16:24]
# convert counter to long and shift bytes
k, iv, meta_mac = self.getCipherKey(key)
ctr = Counter.new(128, initial_value=long(n.encode("hex"), 16) << 64)
cipher = AES.new(k, AES.MODE_CTR, counter=ctr)
self.pyfile.setStatus("decrypting")
self.pyfile.setProgress(0)
file_crypted = fs_encode(self.lastDownload)
file_decrypted = file_crypted.rsplit(self.FILE_SUFFIX)[0]
try:
f = open(file_crypted, "rb")
df = open(file_decrypted, "wb")
except IOError, e:
self.fail(e)
chunk_size = 2 ** 15 # buffer size, 32k
# file_mac = [0, 0, 0, 0] # calculate CBC-MAC for checksum
chunks = os.path.getsize(file_crypted) / chunk_size + 1
for i in xrange(chunks):
buf = f.read(chunk_size)
if not buf:
break
chunk = cipher.decrypt(buf)
df.write(chunk)
self.pyfile.setProgress(int((100.0 / chunks) * i))
# chunk_mac = [iv[0], iv[1], iv[0], iv[1]]
# for i in xrange(0, chunk_size, 16):
# block = chunk[i:i+16]
# if len(block) % 16:
# block += '=' * (16 - (len(block) % 16))
# block = array.array("I", block)
# chunk_mac = [chunk_mac[0] ^ a_[0], chunk_mac[1] ^ block[1], chunk_mac[2] ^ block[2], chunk_mac[3] ^ block[3]]
# chunk_mac = aes_cbc_encrypt_a32(chunk_mac, k)
# file_mac = [file_mac[0] ^ chunk_mac[0], file_mac[1] ^ chunk_mac[1], file_mac[2] ^ chunk_mac[2], file_mac[3] ^ chunk_mac[3]]
# file_mac = aes_cbc_encrypt_a32(file_mac, k)
self.pyfile.setProgress(100)
f.close()
df.close()
# if file_mac[0] ^ file_mac[1], file_mac[2] ^ file_mac[3] != meta_mac:
# os.remove(file_decrypted)
# self.fail(_("Checksum mismatch"))
os.remove(file_crypted)
self.lastDownload = fs_decode(file_decrypted)
def download(self,
url,
get={},
post={},
ref=True,
cookies=True,
disposition=False):
"""Downloads the content at url to download folder
:param url:
:param get:
:param post:
:param ref:
:param cookies:
:param disposition: if True and server provides content-disposition header\
the filename will be changed if needed
:return: The location where the file was saved
"""
self.checkForSameFiles()
self.pyfile.setStatus("downloading")
download_folder = self.config['general']['download_folder']
location = save_join(download_folder, self.pyfile.package().folder)
if not exists(location):
makedirs(location, int(self.core.config["permission"]["folder"],
8))
if self.core.config["permission"]["change_dl"] and not IS_WINDOWS:
try:
uid = getpwnam(self.config["permission"]["user"])[2]
gid = getgrnam(self.config["permission"]["group"])[2]
chown(location, uid, gid)
except Exception as e:
self.log.warning(
_("Setting User and Group failed: %s") % str(e))
# convert back to unicode
location = fs_decode(location)
name = save_path(self.pyfile.name)
filename = join(location, name)
get_hook_manager().dispatchEvent("downloadStarts", self.pyfile, url,
filename)
try:
newname = self.req.httpDownload(
url,
filename,
get=get,
post=post,
ref=ref,
cookies=cookies,
chunks=self.getChunkCount(),
resume=self.resumeDownload,
progressNotify=self.pyfile.setProgress,
disposition=disposition)
finally:
self.pyfile.size = self.req.size
if disposition and newname and newname != name: #triple check, just to be sure
self.log.info("%(name)s saved as %(newname)s" % {
"name": name,
"newname": newname
})
self.pyfile.name = newname
filename = join(location, newname)
fs_filename = fs_encode(filename)
if self.core.config["permission"]["change_file"]:
chmod(fs_filename, int(self.core.config["permission"]["file"], 8))
if self.core.config["permission"]["change_dl"] and not IS_WINDOWS:
try:
uid = getpwnam(self.config["permission"]["user"])[2]
gid = getgrnam(self.config["permission"]["group"])[2]
chown(fs_filename, uid, gid)
except Exception as e:
self.log.warning(
_("Setting User and Group failed: %s") % str(e))
self.lastDownload = filename
return self.lastDownload
class Plugin(Base):
__name__ = "Plugin"
__type__ = "hoster"
__version__ = "0.10"
__pattern__ = r'^unmatchable$'
__config__ = [] #: [("name", "type", "desc", "default")]
__description__ = """Base plugin"""
__license__ = "GPLv3"
__authors__ = [("RaNaN", "*****@*****.**"),
("spoob", "*****@*****.**"), ("mkaay", "*****@*****.**")]
def __init__(self, pyfile):
Base.__init__(self, pyfile.m.core)
self.wantReconnect = False
#: enables simultaneous processing of multiple downloads
self.multiDL = True
self.limitDL = 0
#: chunk limit
self.chunkLimit = 1
self.resumeDownload = False
#: time() + wait in seconds
self.waitUntil = 0
self.waiting = False
self.ocr = None #captcha reader instance
#: account handler instance, see :py:class:`Account`
self.account = pyfile.m.core.accountManager.getAccountPlugin(
self.__name__)
#: premium status
self.premium = False
#: username/login
self.user = None
if self.account and not self.account.canUse(): self.account = None
if self.account:
self.user, data = self.account.selectAccount()
#: Browser instance, see `network.Browser`
self.req = self.account.getAccountRequest(self.user)
self.chunkLimit = -1 # chunk limit, -1 for unlimited
#: enables resume (will be ignored if server dont accept chunks)
self.resumeDownload = True
self.multiDL = True #every hoster with account should provide multiple downloads
#: premium status
self.premium = self.account.isPremium(self.user)
else:
self.req = pyfile.m.core.requestFactory.getRequest(self.__name__)
#: associated pyfile instance, see `PyFile`
self.pyfile = pyfile
self.thread = None # holds thread in future
#: location where the last call to download was saved
self.lastDownload = ""
#: re match of the last call to `checkDownload`
self.lastCheck = None
#: js engine, see `JsEngine`
self.js = self.core.js
self.cTask = None #captcha task
self.retries = 0 # amount of retries already made
self.html = None # some plugins store html code here
self.init()
def getChunkCount(self):
if self.chunkLimit <= 0:
return self.config["download"]["chunks"]
return min(self.config["download"]["chunks"], self.chunkLimit)
def __call__(self):
return self.__name__
def init(self):
"""initialize the plugin (in addition to `__init__`)"""
pass
def setup(self):
""" setup for enviroment and other things, called before downloading (possibly more than one time)"""
pass
def preprocessing(self, thread):
""" handles important things to do before starting """
self.thread = thread
if self.account:
self.account.checkLogin(self.user)
else:
self.req.clearCookies()
self.setup()
self.pyfile.setStatus("starting")
return self.process(self.pyfile)
def process(self, pyfile):
"""the 'main' method of every plugin, you **have to** overwrite it"""
raise NotImplementedError
def resetAccount(self):
""" dont use account and retry download """
self.account = None
self.req = self.core.requestFactory.getRequest(self.__name__)
self.retry()
def checksum(self, local_file=None):
"""
return codes:
0 - checksum ok
1 - checksum wrong
5 - can't get checksum
10 - not implemented
20 - unknown error
"""
#@TODO checksum check hook
return True, 10
def setWait(self, seconds, reconnect=False):
"""Set a specific wait time later used with `wait`
:param seconds: wait time in seconds
:param reconnect: True if a reconnect would avoid wait time
"""
if reconnect:
self.wantReconnect = True
self.pyfile.waitUntil = time() + int(seconds)
def wait(self):
""" waits the time previously set """
self.waiting = True
self.pyfile.setStatus("waiting")
while self.pyfile.waitUntil > time():
self.thread.m.reconnecting.wait(2)
if self.pyfile.abort: raise Abort
if self.thread.m.reconnecting.isSet():
self.waiting = False
self.wantReconnect = False
raise Reconnect
self.waiting = False
self.pyfile.setStatus("starting")
def offline(self):
""" fail and indicate file is offline """
raise Fail("offline")
def tempOffline(self):
""" fail and indicates file ist temporary offline, the core may take consequences """
raise Fail("temp. offline")
def skip(self, reason):
raise Skip(reason)
def retry(self, max_tries=3, wait_time=1, reason=""):
"""Retries and begin again from the beginning
:param max_tries: number of maximum retries
:param wait_time: time to wait in seconds
:param reason: reason for retrying, will be passed to fail if max_tries reached
"""
if 0 < max_tries <= self.retries:
if not reason: reason = "Max retries reached"
raise Fail(reason)
self.wantReconnect = False
self.setWait(wait_time)
self.wait()
self.retries += 1
raise Retry(reason)
def invalidCaptcha(self):
if self.cTask:
self.cTask.invalid()
def correctCaptcha(self):
if self.cTask:
self.cTask.correct()
def decryptCaptcha(self,
url,
get={},
post={},
cookies=False,
forceUser=False,
imgtype='jpg',
result_type='textual'):
""" Loads a captcha and decrypts it with ocr, plugin, user input
:param url: url of captcha image
:param get: get part for request
:param post: post part for request
:param cookies: True if cookies should be enabled
:param forceUser: if True, ocr is not used
:param imgtype: Type of the Image
:param result_type: 'textual' if text is written on the captcha\
or 'positional' for captcha where the user have to click\
on a specific region on the captcha
:return: result of decrypting
"""
img = self.load(url, get=get, post=post, cookies=cookies)
id = ("%.2f" % time())[-6:].replace(".", "")
temp_file = open(
join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)),
"wb")
temp_file.write(img)
temp_file.close()
has_plugin = self.__name__ in self.core.pluginManager.captchaPlugins
if self.core.captcha:
Ocr = self.core.pluginManager.loadClass("captcha", self.__name__)
else:
Ocr = None
if Ocr and not forceUser:
sleep(randint(3000, 5000) / 1000.0)
if self.pyfile.abort: raise Abort
ocr = Ocr()
result = ocr.get_captcha(temp_file.name)
else:
captchaManager = self.core.captchaManager
task = captchaManager.newTask(img, imgtype, temp_file.name,
result_type)
self.cTask = task
captchaManager.handleCaptcha(task)
while task.isWaiting():
if self.pyfile.abort:
captchaManager.removeTask(task)
raise Abort
sleep(1)
captchaManager.removeTask(task)
if task.error and has_plugin: #ignore default error message since the user could use OCR
self.fail(
_("Pil and tesseract not installed and no Client connected for captcha decrypting"
))
elif task.error:
self.fail(task.error)
elif not task.result:
self.fail(
_("No captcha result obtained in appropiate time by any of the plugins."
))
result = task.result
self.log.debug("Received captcha result: %s" % str(result))
if not self.core.debug:
try:
remove(temp_file.name)
except:
pass
return result
def load(self,
url,
get={},
post={},
ref=True,
cookies=True,
just_header=False,
decode=False):
"""Load content at url and returns it
:param url:
:param get:
:param post:
:param ref:
:param cookies:
:param just_header: if True only the header will be retrieved and returned as dict
:param decode: Wether to decode the output according to http header, should be True in most cases
:return: Loaded content
"""
if self.pyfile.abort: raise Abort
#utf8 vs decode -> please use decode attribute in all future plugins
if type(url) == unicode: url = str(url)
res = self.req.load(url,
get,
post,
ref,
cookies,
just_header,
decode=decode)
if self.core.debug:
from inspect import currentframe
frame = currentframe()
if not exists(join("tmp", self.__name__)):
makedirs(join("tmp", self.__name__))
f = open(
join(
"tmp", self.__name__, "%s_line%s.dump.html" %
(frame.f_back.f_code.co_name, frame.f_back.f_lineno)),
"wb")
del frame # delete the frame or it wont be cleaned
try:
tmp = res.encode("utf8")
except:
tmp = res
f.write(tmp)
f.close()
if just_header:
#parse header
header = {"code": self.req.code}
for line in res.splitlines():
line = line.strip()
if not line or ":" not in line: continue
key, none, value = line.partition(":")
key = key.lower().strip()
value = value.strip()
if key in header:
if type(header[key]) == list:
header[key].append(value)
else:
header[key] = [header[key], value]
else:
header[key] = value
res = header
return res
def download(self,
url,
get={},
post={},
ref=True,
cookies=True,
disposition=False):
"""Downloads the content at url to download folder
:param url:
:param get:
:param post:
:param ref:
:param cookies:
:param disposition: if True and server provides content-disposition header\
the filename will be changed if needed
:return: The location where the file was saved
"""
self.checkForSameFiles()
self.pyfile.setStatus("downloading")
download_folder = self.config['general']['download_folder']
location = save_join(download_folder, self.pyfile.package().folder)
if not exists(location):
makedirs(location, int(self.core.config["permission"]["folder"],
8))
if self.core.config["permission"]["change_dl"] and os.name != "nt":
try:
uid = getpwnam(self.config["permission"]["user"])[2]
gid = getgrnam(self.config["permission"]["group"])[2]
chown(location, uid, gid)
except Exception, e:
self.log.warning(
_("Setting User and Group failed: %s") % str(e))
# convert back to unicode
location = fs_decode(location)
name = save_path(self.pyfile.name)
filename = join(location, name)
self.core.hookManager.dispatchEvent("downloadStarts", self.pyfile, url,
filename)
try:
newname = self.req.httpDownload(
url,
filename,
get=get,
post=post,
ref=ref,
cookies=cookies,
chunks=self.getChunkCount(),
resume=self.resumeDownload,
progressNotify=self.pyfile.setProgress,
disposition=disposition)
finally:
self.pyfile.size = self.req.size
if disposition and newname and newname != name: #triple check, just to be sure
self.log.info("%(name)s saved as %(newname)s" % {
"name": name,
"newname": newname
})
self.pyfile.name = newname
filename = join(location, newname)
fs_filename = fs_encode(filename)
if self.core.config["permission"]["change_file"]:
chmod(fs_filename, int(self.core.config["permission"]["file"], 8))
if self.core.config["permission"]["change_dl"] and os.name != "nt":
try:
uid = getpwnam(self.config["permission"]["user"])[2]
gid = getgrnam(self.config["permission"]["group"])[2]
chown(fs_filename, uid, gid)
except Exception, e:
self.log.warning(
_("Setting User and Group failed: %s") % str(e))
