def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False, follow_location=True, save_cookies=True):
"""Load content at url and returns it
:param url:
:param get:
:param post:
:param ref:
:param cookies:
:param just_header: If True only the header will be retrieved and returned as dict
:param decode: Wether to decode the output according to http header, should be True in most cases
:param follow_location: If True follow location else not
:param save_cookies: If True saves received cookies else discard them
:return: Loaded content
"""
if self.pyfile.abort:
self.abort()
if not url:
self.fail(_("No url given"))
url = urllib.unquote(encode(url).strip()) #@NOTE: utf8 vs decode -> please use decode attribute in all future plugins
if self.core.debug:
self.log_debug("Load url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")])
res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode, follow_location=follow_location, save_cookies=save_cookies)
if decode:
res = encode(res)
if self.core.debug:
import inspect
frame = inspect.currentframe()
framefile = fs_join("tmp", self.get_class_name(), "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno))
try:
if not os.path.exists(os.path.join("tmp", self.get_class_name())):
os.makedirs(os.path.join("tmp", self.get_class_name()))
with open(framefile, "wb") as f:
del frame #: delete the frame or it wont be cleaned
f.write(res)
except IOError, e:
self.log_error(e)
def set_request_context(self, url, get, post, referer, cookies, multipart=False):
"""Sets everything needed for the request"""
url = urllib.quote(encode(url).strip(), safe="%/:=&?~#+!$,;'@()*[]") #@TODO: recheck
if get:
get = urllib.urlencode(get)
url = "%s?%s" % (url, get)
self.c.setopt(pycurl.URL, url)
self.c.lastUrl = url
if post:
self.c.setopt(pycurl.POST, 1)
if not multipart:
if type(post) == unicode:
post = str(post) #: unicode not allowed
elif type(post) == str:
pass
else:
post = urllib.urlencode(dict((encode(x), encode(y)) for x, y in dict(post).iteritems()))
self.c.setopt(pycurl.POSTFIELDS, post)
else:
post = [(x, encode(y)) for x, y in post.iteritems()]
self.c.setopt(pycurl.HTTPPOST, post)
else:
self.c.setopt(pycurl.POST, 0)
if referer and self.lastURL:
self.c.setopt(pycurl.REFERER, str(self.lastURL))
if cookies:
self.c.setopt(pycurl.COOKIEFILE, "")
self.c.setopt(pycurl.COOKIEJAR, "")
self.get_cookies()
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False):
"""Downloads the content at url to download folder
:param url:
:param get:
:param post:
:param ref:
:param cookies:
:param disposition: if True and server provides content-disposition header\
the filename will be changed if needed
:return: The location where the file was saved
"""
if self.pyfile.abort:
self.abort()
if not url:
self.fail(_("No url given"))
url = urllib.unquote(encode(url).strip())
if self.core.debug:
self.log_debug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")])
self.check_for_same_files()
self.pyfile.set_status("downloading")
if disposition:
self.pyfile.name = urlparse.urlparse(url).path.split('/')[-1] or self.pyfile.name
download_folder = self.core.config.get("general", "download_folder")
location = fs_join(download_folder, self.pyfile.package().folder)
if not os.path.exists(location):
try:
os.makedirs(location, int(self.core.config.get("permission", "folder"), 8))
if self.core.config.get("permission", "change_dl") and os.name != "nt":
uid = pwd.getpwnam(self.core.config.get("permission", "user"))[2]
gid = grp.getgrnam(self.core.config.get("permission", "group"))[2]
os.chown(location, uid, gid)
except Exception, e:
self.fail(e)
def cast(self, typ, value):
"""Cast value to given format"""
if not isinstance(value, basestring):
return value
elif typ == "int":
return int(value)
elif typ == "bool":
return value.lower() in ("1", "true", "on", "an", "yes")
elif typ == "time":
if not value:
value = "0:00"
if not ":" in value:
value += ":00"
return value
elif typ in ("str", "file", "folder"):
return encode(value)
else:
return value
def save(self):
"""Saves config to filename"""
configs = []
f = open(self.CONFIG, "wb")
configs.append(f)
chmod(self.CONFIG, 0600)
f.write("version: %i\n\n" % CONF_VERSION)
for section, data in self.config.iteritems():
f.write("[%s]\n" % section)
for option, data in data.config.iteritems():
value = encode(self.get(section, option))
f.write('%s = %s\n' % (option, value))
f.write("\n")
f.close()
def __init__(self, url, filename, get={}, post={}, referer=None, cj=None, bucket=None,
options={}, progress=None, disposition=False):
self.url = urllib.unquote(encode(url).strip())
self.filename = filename.strip() #: complete file destination, not only name
self.get = get
self.post = post
self.referer = referer
self.cj = cj #: cookiejar if cookies are needed
self.bucket = bucket
self.options = options
self.disposition = disposition
# all arguments
self.abort = False
self.size = 0
self.nameDisposition = None #: will be parsed from content disposition
self.chunks = []
self.log = logging.get_logger("log")
try:
self.info = ChunkInfo.load(filename)
self.info.resume = True #: resume is only possible with valid info file
self.size = self.info.size
self.infoSaved = True
except IOError:
self.info = Chunk_info(filename)
self.chunkSupport = True
self.manager = pycurl.Curl_multi()
# needed for speed calculation
self.lastArrived = []
self.speeds = []
self.lastSpeeds = [0, 0]
self.progress = progress
def get_plugin(self, plugin, option):
"""Gets a value for a plugin"""
value = self.plugin[plugin][option]["value"]
return encode(value)
def save_config(self, config, filename):
"""Saves config to filename"""
with open(filename, "wb") as f:
try:
os.chmod(filename, 0600)
except Exception:
pass
f.write("version: %i \n" % CONF_VERSION)
for section in config.iterkeys():
f.write('\n%s - "%s":\n' % (section, config[section]["desc"]))
for option, data in sorted(
config[section].items(), key=lambda i: i[1]["idx"] if i[0] not in ("desc", "outline") else 0
):
if option in ("desc", "outline"):
continue
if isinstance(data["value"], list):
value = "[ \n"
for x in data["value"]:
value += "\t\t" + str(x) + ",\n"
value += "\t\t]\n"
else:
if isinstance(data["value"], basestring):
value = data["value"] + "\n"
else:
value = str(data["value"]) + "\n"
try:
f.write('\t%s %s : "%s" = %s' % (data["type"], option, data["desc"], value))
except UnicodeEncodeError:
f.write('\t%s %s : "%s" = %s' % (data["type"], option, data["desc"], encode(value)))
def path(file="", path=""):
type = "file" if file else "folder"
path = os.path.normpath(unquotepath(path))
if os.path.isfile(path):
oldfile = path
path = os.path.dirname(path)
else:
oldfile = ''
abs = False
if os.path.isdir(path):
if os.path.isabs(path):
cwd = os.path.abspath(path)
abs = True
else:
cwd = os.relpath(path)
else:
cwd = os.getcwd()
cwd = os.path.normpath(os.path.abspath(encode(cwd)))
parentdir = os.path.dirname(cwd)
if not abs:
if os.path.abspath(cwd) == "/":
cwd = os.relpath(cwd)
else:
cwd = os.relpath(cwd) + os.path.sep
parentdir = os.relpath(parentdir) + os.path.sep
if os.path.abspath(cwd) == "/":
parentdir = ""
try:
folders = os.listdir(cwd)
except Exception:
folders = []
files = []
for f in folders:
try:
f = f.decode(sys.getfilesystemencoding())
data = {'name': f, 'fullpath': os.path.join(cwd, f)}
data['sort'] = data['fullpath'].lower()
data['modified'] = datetime.datetime.fromtimestamp(int(os.path.getmtime(os.path.join(cwd, f))))
data['ext'] = os.path.splitext(f)[1]
except Exception:
continue
data['type'] = 'dir' if os.path.isdir(os.path.join(cwd, f)) else 'file'
if os.path.isfile(os.path.join(cwd, f)):
data['size'] = os.path.getsize(os.path.join(cwd, f))
power = 0
while (data['size'] / 1024) > 0.3:
power += 1
data['size'] /= 1024.
units = ('', 'K', 'M', 'G', 'T')
data['unit'] = units[power] + 'Byte'
else:
data['size'] = ''
files.append(data)
files = sorted(files, key=operator.itemgetter('type', 'sort'))
return render_to_response('pathchooser.html',
{'cwd': cwd, 'files': files, 'parentdir': parentdir, 'type': type, 'oldfile': oldfile,
'absolute': abs}, [])
def write_string(self, str):
str = encode(str)
self.write_i32(len(str))
self.trans.write(str)
def get_result(self):
return encode(self.result)
def _log(self, type, args):
msg = " | ".join([encode(str(a)).strip() for a in args if a])
logger = getattr(self.core.log, type)
logger("%s: %s" % (self.get_class_name(), msg or _("%s MARK" % type.upper())))
