def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False, follow_location=True, save_cookies=True): """Load content at url and returns it :param url: :param get: :param post: :param ref: :param cookies: :param just_header: If True only the header will be retrieved and returned as dict :param decode: Wether to decode the output according to http header, should be True in most cases :param follow_location: If True follow location else not :param save_cookies: If True saves received cookies else discard them :return: Loaded content """ if self.pyfile.abort: self.abort() if not url: self.fail(_("No url given")) url = urllib.unquote(encode(url).strip()) #@NOTE: utf8 vs decode -> please use decode attribute in all future plugins if self.core.debug: self.log_debug("Load url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode, follow_location=follow_location, save_cookies=save_cookies) if decode: res = encode(res) if self.core.debug: import inspect frame = inspect.currentframe() framefile = fs_join("tmp", self.get_class_name(), "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) try: if not os.path.exists(os.path.join("tmp", self.get_class_name())): os.makedirs(os.path.join("tmp", self.get_class_name())) with open(framefile, "wb") as f: del frame #: delete the frame or it wont be cleaned f.write(res) except IOError, e: self.log_error(e)
def set_request_context(self, url, get, post, referer, cookies, multipart=False): """Sets everything needed for the request""" url = urllib.quote(encode(url).strip(), safe="%/:=&?~#+!$,;'@()*[]") #@TODO: recheck if get: get = urllib.urlencode(get) url = "%s?%s" % (url, get) self.c.setopt(pycurl.URL, url) self.c.lastUrl = url if post: self.c.setopt(pycurl.POST, 1) if not multipart: if type(post) == unicode: post = str(post) #: unicode not allowed elif type(post) == str: pass else: post = urllib.urlencode(dict((encode(x), encode(y)) for x, y in dict(post).iteritems())) self.c.setopt(pycurl.POSTFIELDS, post) else: post = [(x, encode(y)) for x, y in post.iteritems()] self.c.setopt(pycurl.HTTPPOST, post) else: self.c.setopt(pycurl.POST, 0) if referer and self.lastURL: self.c.setopt(pycurl.REFERER, str(self.lastURL)) if cookies: self.c.setopt(pycurl.COOKIEFILE, "") self.c.setopt(pycurl.COOKIEJAR, "") self.get_cookies()
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): """Downloads the content at url to download folder :param url: :param get: :param post: :param ref: :param cookies: :param disposition: if True and server provides content-disposition header\ the filename will be changed if needed :return: The location where the file was saved """ if self.pyfile.abort: self.abort() if not url: self.fail(_("No url given")) url = urllib.unquote(encode(url).strip()) if self.core.debug: self.log_debug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) self.check_for_same_files() self.pyfile.set_status("downloading") if disposition: self.pyfile.name = urlparse.urlparse(url).path.split('/')[-1] or self.pyfile.name download_folder = self.core.config.get("general", "download_folder") location = fs_join(download_folder, self.pyfile.package().folder) if not os.path.exists(location): try: os.makedirs(location, int(self.core.config.get("permission", "folder"), 8)) if self.core.config.get("permission", "change_dl") and os.name != "nt": uid = pwd.getpwnam(self.core.config.get("permission", "user"))[2] gid = grp.getgrnam(self.core.config.get("permission", "group"))[2] os.chown(location, uid, gid) except Exception, e: self.fail(e)
def cast(self, typ, value): """Cast value to given format""" if not isinstance(value, basestring): return value elif typ == "int": return int(value) elif typ == "bool": return value.lower() in ("1", "true", "on", "an", "yes") elif typ == "time": if not value: value = "0:00" if not ":" in value: value += ":00" return value elif typ in ("str", "file", "folder"): return encode(value) else: return value
def save(self): """Saves config to filename""" configs = [] f = open(self.CONFIG, "wb") configs.append(f) chmod(self.CONFIG, 0600) f.write("version: %i\n\n" % CONF_VERSION) for section, data in self.config.iteritems(): f.write("[%s]\n" % section) for option, data in data.config.iteritems(): value = encode(self.get(section, option)) f.write('%s = %s\n' % (option, value)) f.write("\n") f.close()
def __init__(self, url, filename, get={}, post={}, referer=None, cj=None, bucket=None, options={}, progress=None, disposition=False): self.url = urllib.unquote(encode(url).strip()) self.filename = filename.strip() #: complete file destination, not only name self.get = get self.post = post self.referer = referer self.cj = cj #: cookiejar if cookies are needed self.bucket = bucket self.options = options self.disposition = disposition # all arguments self.abort = False self.size = 0 self.nameDisposition = None #: will be parsed from content disposition self.chunks = [] self.log = logging.get_logger("log") try: self.info = ChunkInfo.load(filename) self.info.resume = True #: resume is only possible with valid info file self.size = self.info.size self.infoSaved = True except IOError: self.info = Chunk_info(filename) self.chunkSupport = True self.manager = pycurl.Curl_multi() # needed for speed calculation self.lastArrived = [] self.speeds = [] self.lastSpeeds = [0, 0] self.progress = progress
def get_plugin(self, plugin, option): """Gets a value for a plugin""" value = self.plugin[plugin][option]["value"] return encode(value)
def save_config(self, config, filename): """Saves config to filename""" with open(filename, "wb") as f: try: os.chmod(filename, 0600) except Exception: pass f.write("version: %i \n" % CONF_VERSION) for section in config.iterkeys(): f.write('\n%s - "%s":\n' % (section, config[section]["desc"])) for option, data in sorted( config[section].items(), key=lambda i: i[1]["idx"] if i[0] not in ("desc", "outline") else 0 ): if option in ("desc", "outline"): continue if isinstance(data["value"], list): value = "[ \n" for x in data["value"]: value += "\t\t" + str(x) + ",\n" value += "\t\t]\n" else: if isinstance(data["value"], basestring): value = data["value"] + "\n" else: value = str(data["value"]) + "\n" try: f.write('\t%s %s : "%s" = %s' % (data["type"], option, data["desc"], value)) except UnicodeEncodeError: f.write('\t%s %s : "%s" = %s' % (data["type"], option, data["desc"], encode(value)))
def path(file="", path=""): type = "file" if file else "folder" path = os.path.normpath(unquotepath(path)) if os.path.isfile(path): oldfile = path path = os.path.dirname(path) else: oldfile = '' abs = False if os.path.isdir(path): if os.path.isabs(path): cwd = os.path.abspath(path) abs = True else: cwd = os.relpath(path) else: cwd = os.getcwd() cwd = os.path.normpath(os.path.abspath(encode(cwd))) parentdir = os.path.dirname(cwd) if not abs: if os.path.abspath(cwd) == "/": cwd = os.relpath(cwd) else: cwd = os.relpath(cwd) + os.path.sep parentdir = os.relpath(parentdir) + os.path.sep if os.path.abspath(cwd) == "/": parentdir = "" try: folders = os.listdir(cwd) except Exception: folders = [] files = [] for f in folders: try: f = f.decode(sys.getfilesystemencoding()) data = {'name': f, 'fullpath': os.path.join(cwd, f)} data['sort'] = data['fullpath'].lower() data['modified'] = datetime.datetime.fromtimestamp(int(os.path.getmtime(os.path.join(cwd, f)))) data['ext'] = os.path.splitext(f)[1] except Exception: continue data['type'] = 'dir' if os.path.isdir(os.path.join(cwd, f)) else 'file' if os.path.isfile(os.path.join(cwd, f)): data['size'] = os.path.getsize(os.path.join(cwd, f)) power = 0 while (data['size'] / 1024) > 0.3: power += 1 data['size'] /= 1024. units = ('', 'K', 'M', 'G', 'T') data['unit'] = units[power] + 'Byte' else: data['size'] = '' files.append(data) files = sorted(files, key=operator.itemgetter('type', 'sort')) return render_to_response('pathchooser.html', {'cwd': cwd, 'files': files, 'parentdir': parentdir, 'type': type, 'oldfile': oldfile, 'absolute': abs}, [])
def write_string(self, str): str = encode(str) self.write_i32(len(str)) self.trans.write(str)
def get_result(self): return encode(self.result)
def _log(self, type, args): msg = " | ".join([encode(str(a)).strip() for a in args if a]) logger = getattr(self.core.log, type) logger("%s: %s" % (self.get_class_name(), msg or _("%s MARK" % type.upper())))