def download(self,
url,
get={},
post={},
ref=True,
cookies=True,
disposition=False):
"""Downloads the content at url to download folder
:param url:
:param get:
:param post:
:param ref:
:param cookies:
:param disposition: if True and server provides content-disposition header\
the filename will be changed if needed
:return: The location where the file was saved
"""
if self.pyfile.abort:
self.abort()
if not url:
self.fail(_("No url given"))
url = urllib.unquote(encode(url).strip())
if self.core.debug:
self.logDebug(
"Download url: " + url, *[
"%s=%s" % (key, val) for key, val in locals().iteritems()
if key not in ("self", "url")
])
self.checkForSameFiles()
self.pyfile.setStatus("downloading")
if disposition:
self.pyfile.name = urlparse.urlparse(url).path.split(
'/')[-1] or self.pyfile.name
download_folder = self.core.config.get("general", "download_folder")
location = fs_join(download_folder, self.pyfile.package().folder)
if not exists(location):
try:
makedirs(location,
int(self.core.config.get("permission", "folder"), 8))
if self.core.config.get("permission",
"change_dl") and os.name != "nt":
uid = getpwnam(self.core.config.get("permission",
"user"))[2]
gid = getgrnam(self.core.config.get("permission",
"group"))[2]
chown(location, uid, gid)
except Exception, e:
self.fail(e)
def eval(self, script, engine=None): #: engine can be a jse name """string""" or an AbstractEngine """class"""
JSE = self.get(engine)
if not JSE:
raise TypeError("engine")
script = encode(script)
out, err = JSE.eval(script)
results = [out]
if self.core.config.get("general", "debug"):
if err:
self.core.log.debug(JSE._name + ":", err)
engines = self.find()
engines.remove(JSE)
for E in engines:
out, err = E.eval(script)
res = err or out
self.core.log.debug(E._name + ":", res)
results.append(res)
if len(results) > 1 and len(uniqify(results)) > 1:
self.core.log.warning("JS output of two or more engines mismatch")
return results[0]
def saveConfig(self, config, filename):
"""saves config to filename"""
with open(filename, "wb") as f:
chmod(filename, 0600)
f.write("version: %i \n" % CONF_VERSION)
for section in config.iterkeys():
f.write('\n%s - "%s":\n' % (section, config[section]["desc"]))
for option, data in config[section].iteritems():
if option in ("desc", "outline"): continue
if isinstance(data["value"], list):
value = "[ \n"
for x in data["value"]:
value += "\t\t" + str(x) + ",\n"
value += "\t\t]\n"
else:
if isinstance(data["value"], basestring):
value = data["value"] + "\n"
else:
value = str(data["value"]) + "\n"
try:
f.write('\t%s %s : "%s" = %s' %
(data["type"], option, data["desc"], value))
except UnicodeEncodeError:
f.write('\t%s %s : "%s" = %s' %
(data["type"], option, data["desc"],
encode(value)))
def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False, follow_location=True, save_cookies=True):
"""Load content at url and returns it
:param url:
:param get:
:param post:
:param ref:
:param cookies:
:param just_header: If True only the header will be retrieved and returned as dict
:param decode: Wether to decode the output according to http header, should be True in most cases
:param follow_location: If True follow location else not
:param save_cookies: If True saves received cookies else discard them
:return: Loaded content
"""
if self.pyfile.abort:
self.abort()
if not url:
self.fail(_("No url given"))
url = urllib.unquote(encode(url).strip()) #@NOTE: utf8 vs decode -> please use decode attribute in all future plugins
if self.core.debug:
self.logDebug("Load url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")])
res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode, follow_location=follow_location, save_cookies=save_cookies)
if decode:
res = encode(res)
if self.core.debug:
import inspect
frame = inspect.currentframe()
framefile = fs_join("tmp", self.getClassName(), "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno))
try:
if not os.path.exists(os.path.join("tmp", self.getClassName())):
os.makedirs(os.path.join("tmp", self.getClassName()))
with open(framefile, "wb") as f:
del frame #: delete the frame or it wont be cleaned
f.write(res)
except IOError, e:
self.logError(e)
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False):
"""Downloads the content at url to download folder
:param url:
:param get:
:param post:
:param ref:
:param cookies:
:param disposition: if True and server provides content-disposition header\
the filename will be changed if needed
:return: The location where the file was saved
"""
if self.pyfile.abort:
self.abort()
if not url:
self.fail(_("No url given"))
url = urllib.unquote(encode(url).strip())
if self.core.debug:
self.logDebug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")])
self.checkForSameFiles()
self.pyfile.setStatus("downloading")
if disposition:
self.pyfile.name = urlparse.urlparse(url).path.split('/')[-1] or self.pyfile.name
download_folder = self.core.config.get("general", "download_folder")
location = fs_join(download_folder, self.pyfile.package().folder)
if not os.path.exists(location):
try:
os.makedirs(location, int(self.core.config.get("permission", "folder"), 8))
if self.core.config.get("permission", "change_dl") and os.name != "nt":
uid = pwd.getpwnam(self.core.config.get("permission", "user"))[2]
gid = grp.getgrnam(self.core.config.get("permission", "group"))[2]
os.chown(location, uid, gid)
except Exception, e:
self.fail(e)
def cast(self, typ, value):
"""cast value to given format"""
if not isinstance(value, basestring):
return value
elif typ == "int":
return int(value)
elif typ == "bool":
return value.lower() in ("1", "true", "on", "an", "yes")
elif typ == "time":
if not value:
value = "0:00"
if not ":" in value:
value += ":00"
return value
elif typ in ("str", "file", "folder"):
return encode(value)
else:
return value
def cast(self, typ, value):
"""cast value to given format"""
if not isinstance(value, basestring):
return value
elif typ == "int":
return int(value)
elif typ == "bool":
return value.lower() in ("1", "true", "on", "an", "yes")
elif typ == "time":
if not value:
value = "0:00"
if not ":" in value:
value += ":00"
return value
elif typ in ("str", "file", "folder"):
return encode(value)
else:
return value
def setRequestContext(self,
url,
get,
post,
referer,
cookies,
multipart=False):
""" sets everything needed for the request """
url = myquote(url)
if get:
get = urlencode(get)
url = "%s?%s" % (url, get)
self.c.setopt(pycurl.URL, url)
self.c.lastUrl = url
if post:
self.c.setopt(pycurl.POST, 1)
if not multipart:
if type(post) == unicode:
post = str(post) # unicode not allowed
elif type(post) == str:
pass
else:
post = myurlencode(post)
self.c.setopt(pycurl.POSTFIELDS, post)
else:
post = [(x, encode(y)) for x, y in post.iteritems()]
self.c.setopt(pycurl.HTTPPOST, post)
else:
self.c.setopt(pycurl.POST, 0)
if referer and self.lastURL:
self.c.setopt(pycurl.REFERER, str(self.lastURL))
if cookies:
self.c.setopt(pycurl.COOKIEFILE, "")
self.c.setopt(pycurl.COOKIEJAR, "")
self.getCookies()
def setRequestContext(self, url, get, post, referer, cookies, multipart=False):
""" sets everything needed for the request """
url = myquote(url)
if get:
get = urlencode(get)
url = "%s?%s" % (url, get)
self.c.setopt(pycurl.URL, url)
self.c.lastUrl = url
if post:
self.c.setopt(pycurl.POST, 1)
if not multipart:
if type(post) == unicode:
post = str(post) #: unicode not allowed
elif type(post) == str:
pass
else:
post = myurlencode(post)
self.c.setopt(pycurl.POSTFIELDS, post)
else:
post = [(x, encode(y)) for x, y in post.iteritems()]
self.c.setopt(pycurl.HTTPPOST, post)
else:
self.c.setopt(pycurl.POST, 0)
if referer and self.lastURL:
self.c.setopt(pycurl.REFERER, str(self.lastURL))
if cookies:
self.c.setopt(pycurl.COOKIEFILE, "")
self.c.setopt(pycurl.COOKIEJAR, "")
self.getCookies()
def getPlugin(self, plugin, option):
"""gets a value for a plugin"""
value = self.plugin[plugin][option]["value"]
return encode(value)
def getResult(self):
return encode(self.result)
def myquote(url):
return quote(encode(url), safe="%/:=&?~#+!$,;'@()*[]")
def myurlencode(data):
data = dict(data)
return urlencode(dict((encode(x), encode(y)) for x, y in data.iteritems()))
def saveConfig(self, config, filename):
"""saves config to filename"""
with open(filename, "wb") as f:
chmod(filename, 0600)
f.write("version: %i \n" % CONF_VERSION)
for section in config.iterkeys():
f.write('\n%s - "%s":\n' % (section, config[section]['desc']))
for option, data in sorted(config[section].items(), key=lambda i: i[1]['idx'] if i[0] not in ("desc", "outline") else 0):
if option in ("desc", "outline"):
continue
if isinstance(data['value'], list):
value = "[ \n"
for x in data['value']:
value += "\t\t" + str(x) + ",\n"
value += "\t\t]\n"
else:
if isinstance(data['value'], basestring):
value = data['value'] + "\n"
else:
value = str(data['value']) + "\n"
try:
f.write('\t%s %s : "%s" = %s' % (data['type'], option, data['desc'], value))
except UnicodeEncodeError:
f.write('\t%s %s : "%s" = %s' % (data['type'], option, data['desc'], encode(value)))
def myurlencode(data):
data = dict(data)
return urlencode(dict((encode(x), encode(y)) for x, y in data.iteritems()))
def getResult(self):
return encode(self.result)
def load(self,
url,
get={},
post={},
ref=True,
cookies=True,
just_header=False,
decode=False,
follow_location=True,
save_cookies=True):
"""Load content at url and returns it
:param url:
:param get:
:param post:
:param ref:
:param cookies:
:param just_header: If True only the header will be retrieved and returned as dict
:param decode: Wether to decode the output according to http header, should be True in most cases
:param follow_location: If True follow location else not
:param save_cookies: If True saves received cookies else discard them
:return: Loaded content
"""
if self.pyfile.abort:
self.abort()
if not url:
self.fail(_("No url given"))
url = urllib.unquote(
encode(url).strip()
) #@NOTE: utf8 vs decode -> please use decode attribute in all future plugins
if self.core.debug:
self.logDebug(
"Load url: " + url, *[
"%s=%s" % (key, val) for key, val in locals().iteritems()
if key not in ("self", "url")
])
res = self.req.load(url,
get,
post,
ref,
cookies,
just_header,
decode=decode,
follow_location=follow_location,
save_cookies=save_cookies)
if decode:
res = encode(res)
if self.core.debug:
import inspect
frame = inspect.currentframe()
framefile = fs_join(
"tmp", self.getClassName(), "%s_line%s.dump.html" %
(frame.f_back.f_code.co_name, frame.f_back.f_lineno))
try:
if not os.path.exists(os.path.join("tmp",
self.getClassName())):
os.makedirs(os.path.join("tmp", self.getClassName()))
with open(framefile, "wb") as f:
del frame #: delete the frame or it wont be cleaned
f.write(res)
except IOError, e:
self.logError(e)
def getPlugin(self, plugin, option):
"""gets a value for a plugin"""
value = self.plugin[plugin][option]['value']
return encode(value)
def _log(self, type, args):
msg = " | ".join([encode(str(a)).strip() for a in args if a])
logger = getattr(self.core.log, type)
logger("%s: %s" % (self.getClassName(), msg or _("%s MARK" % type.upper())))
def myquote(url):
return quote(encode(url), safe="%/:=&?~#+!$,;'@()*[]")
def _log(self, type, args):
msg = " | ".join([encode(str(a)).strip() for a in args if a])
logger = getattr(self.core.log, type)
logger("%s: %s" %
(self.getClassName(), msg or _("%s MARK" % type.upper())))
def saveConfig(self, config, filename):
"""saves config to filename"""
with open(filename, "wb") as f:
try:
os.chmod(filename, 0600)
except Exception:
pass
f.write("version: %i \n" % CONF_VERSION)
for section in config.iterkeys():
f.write('\n%s - "%s":\n' % (section, config[section]['desc']))
for option, data in sorted(config[section].items(), key=lambda i: i[1]['idx'] if i[0] not in ("desc", "outline") else 0):
if option in ("desc", "outline"):
continue
if isinstance(data['value'], list):
value = "[ \n"
for x in data['value']:
value += "\t\t" + str(x) + ",\n"
value += "\t\t]\n"
else:
if isinstance(data['value'], basestring):
value = data['value'] + "\n"
else:
value = str(data['value']) + "\n"
try:
f.write('\t%s %s : "%s" = %s' % (data['type'], option, data['desc'], value))
except UnicodeEncodeError:
f.write('\t%s %s : "%s" = %s' % (data['type'], option, data['desc'], encode(value)))