def downloads():
root = API.get_config_value("general", "download_folder")
if not os.path.isdir(root):
return base([_('Download directory not found.')])
data = {
'folder': [],
'files': []
}
items = os.listdir(fs_encode(root))
for item in sorted([fs_decode(x) for x in items]):
if os.path.isdir(fs_join(root, item)):
folder = {
'name': item,
'path': item,
'files': []
}
files = os.listdir(fs_join(root, item))
for file in sorted([fs_decode(x) for x in files]):
try:
if os.path.isfile(fs_join(root, item, file)):
folder['files'].append(file)
except Exception:
pass
data['folder'].append(folder)
elif os.path.isfile(os.path.join(root, item)):
data['files'].append(item)
return render_to_response('downloads.html', {'files': data}, [pre_processor])
def write_debug_report(self, pyfile):
dump_name = "debug_%s_%s.zip" % (pyfile.pluginname, time.strftime("%d-%m-%Y_%H-%M-%S"))
dump = self.get_debug_dump(pyfile)
try:
import zipfile
zip = zipfile.ZipFile(dump_name, "w")
for f in os.listdir(os.path.join("tmp", pyfile.pluginname)):
try:
# avoid encoding errors
zip.write(os.path.join("tmp", pyfile.pluginname, f), fs_join(pyfile.pluginname, f))
except Exception:
pass
info = zipfile.ZipInfo(fs_join(pyfile.pluginname, "debug_Report.txt"), time.gmtime())
info.external_attr = 0644 << 16L #: change permissions
zip.writestr(info, dump)
zip.close()
if not os.stat(dump_name).st_size:
raise Exception("Empty Zipfile")
except Exception, e:
self.pyload.log.debug("Error creating zip file: %s" % e)
dump_name = dump_name.replace(".zip", ".txt")
with open(dump_name, "wb") as f:
f.write(dump)
def run_tesser(self, subset=False, digits=True, lowercase=True, uppercase=True, pagesegmode=None):
# tmpTif = tempfile.NamedTemporaryFile(suffix=".tif")
try:
tmpTif = open(fs_join("tmp", "tmpTif_%s.tif" % self.get_class_name()), "wb")
tmpTif.close()
# tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt")
tmpTxt = open(fs_join("tmp", "tmpTxt_%s.txt" % self.get_class_name()), "wb")
tmpTxt.close()
except IOError, e:
self.log_error(e)
return
def _copy_chunks(self):
init = fs_encode(self.info.get_chunk_name(0)) #: initial chunk name
if self.info.get_count() > 1:
with open(init, "rb+") as fo: #: first chunkfile
for i in xrange(1, self.info.get_count()):
# input file
fo.seek(
self.info.get_chunk_range(i - 1)[1] + 1) #: seek to beginning of chunk, to get rid of overlapping chunks
fname = fs_encode("%s.chunk%d" % (self.filename, i))
with open(fname, "rb") as fi:
buf = 32 * 1024
while True: #: copy in chunks, consumes less memory
data = fi.read(buf)
if not data:
break
fo.write(data)
if fo.tell() < self.info.get_chunk_range(i)[1]:
os.remove(init)
self.info.remove() #: there are probably invalid chunks
raise Exception("Downloaded content was smaller than expected. Try to reduce download connections.")
os.remove(fname) #: remove chunk
if self.nameDisposition and self.disposition:
self.filename = fs_join(os.path.dirname(self.filename), self.nameDisposition)
shutil.move(init, fs_encode(self.filename))
self.info.remove() #: remove info file
def check_for_same_files(self, starting=False):
""" checks if same file was/is downloaded within same package
:param starting: indicates that the current download is going to start
:raises Skip:
"""
pack = self.pyfile.package()
for pyfile in self.core.files.cache.values():
if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder:
if pyfile.status in (0, 12): #: finished or downloading
raise Skip(pyfile.pluginname)
elif pyfile.status in (5, 7) and starting: #: a download is waiting/starting and was appenrently started before
raise Skip(pyfile.pluginname)
download_folder = self.core.config.get("general", "download_folder")
location = fs_join(download_folder, pack.folder, self.pyfile.name)
if starting and self.core.config.get("download", "skip_existing") and os.path.exists(location):
size = os.stat(location).st_size
if size >= self.pyfile.size:
raise Skip("File exists")
pyfile = self.core.db.find_duplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name)
if pyfile:
if os.path.exists(location):
raise Skip(pyfile[0])
self.log_debug("File %s not skipped, because it does not exists." % self.pyfile.name)
def load_to_disk(self):
"""Loads container to disk if its stored remotely and overwrite url,
or check existent on several places at disk"""
if self.pyfile.url.startswith("http"):
self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1]
content = self.load(self.pyfile.url)
self.pyfile.url = fs_join(self.pyload.config.get("general", "download_folder"), self.pyfile.name)
try:
with open(self.pyfile.url, "wb") as f:
f.write(content)
except IOError, e:
self.fail(str(e))
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False):
"""Downloads the content at url to download folder
:param url:
:param get:
:param post:
:param ref:
:param cookies:
:param disposition: if True and server provides content-disposition header\
the filename will be changed if needed
:return: The location where the file was saved
"""
if self.pyfile.abort:
self.abort()
if not url:
self.fail(_("No url given"))
url = urllib.unquote(encode(url).strip())
if self.core.debug:
self.log_debug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")])
self.check_for_same_files()
self.pyfile.set_status("downloading")
if disposition:
self.pyfile.name = urlparse.urlparse(url).path.split('/')[-1] or self.pyfile.name
download_folder = self.core.config.get("general", "download_folder")
location = fs_join(download_folder, self.pyfile.package().folder)
if not os.path.exists(location):
try:
os.makedirs(location, int(self.core.config.get("permission", "folder"), 8))
if self.core.config.get("permission", "change_dl") and os.name != "nt":
uid = pwd.getpwnam(self.core.config.get("permission", "user"))[2]
gid = grp.getgrnam(self.core.config.get("permission", "group"))[2]
os.chown(location, uid, gid)
except Exception, e:
self.fail(e)
def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False, follow_location=True, save_cookies=True):
"""Load content at url and returns it
:param url:
:param get:
:param post:
:param ref:
:param cookies:
:param just_header: If True only the header will be retrieved and returned as dict
:param decode: Wether to decode the output according to http header, should be True in most cases
:param follow_location: If True follow location else not
:param save_cookies: If True saves received cookies else discard them
:return: Loaded content
"""
if self.pyfile.abort:
self.abort()
if not url:
self.fail(_("No url given"))
url = urllib.unquote(encode(url).strip()) #@NOTE: utf8 vs decode -> please use decode attribute in all future plugins
if self.core.debug:
self.log_debug("Load url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")])
res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode, follow_location=follow_location, save_cookies=save_cookies)
if decode:
res = encode(res)
if self.core.debug:
import inspect
frame = inspect.currentframe()
framefile = fs_join("tmp", self.get_class_name(), "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno))
try:
if not os.path.exists(os.path.join("tmp", self.get_class_name())):
os.makedirs(os.path.join("tmp", self.get_class_name()))
with open(framefile, "wb") as f:
del frame #: delete the frame or it wont be cleaned
f.write(res)
except IOError, e:
self.log_error(e)
def remove_plugins(self, type_plugins):
"""Delete plugins from disk"""
if not type_plugins:
return
removed = set()
self.log_debug("Requested deletion of plugins: %s" % type_plugins)
for type, name in type_plugins:
rootplugins = os.path.join(pypath, "module", "plugins")
for dir in ("userplugins", rootplugins):
py_filename = fs_join(dir, type, name + ".py")
pyc_filename = py_filename + "c"
if type == "addon":
try:
self.manager.deactivate_addon(name)
except Exception, e:
self.log_debug(e)
for filename in (py_filename, pyc_filename):
if not exists(filename):
continue
try:
os.remove(filename)
except OSError, e:
self.log_error(_("Error removing: %s") % filename, e)
else:
id = (type, name)
removed.add(id)
self.create_packages()
def load_to_disk(self):
"""Loads container to disk if its stored remotely and overwrite url,
or check existent on several places at disk"""
if self.pyfile.url.startswith("http"):
self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1]
content = self.load(self.pyfile.url)
self.pyfile.url = fs_join(self.pyload.config.get("general", "download_folder"), self.pyfile.name)
try:
with open(self.pyfile.url, "wb") as f:
f.write(content)
except IOError, e:
self.fail(str(e))
else:
self.pyfile.name = os.path.basename(self.pyfile.url)
if not os.path.exists(self.pyfile.url):
if os.path.exists(fs_join(pypath, self.pyfile.url)):
self.pyfile.url = fs_join(pypath, self.pyfile.url)
else:
self.fail(_("File not exists"))
def delete_tmp(self):
if self.pyfile.name.startswith("tmp_"):
os.remove(self.pyfile.url)