def _init_filelogger(self):
fmt = "%(asctime)s %(levelname)-8s %(message)s"
datefmt = "%Y-%m-%d %H:%M:%S"
fileform = logging.Formatter(fmt, datefmt)
logfile_folder = self.config.get('log', 'logfile_folder')
if not logfile_folder:
logfile_folder = self.DEFAULT_LOGDIRNAME
makedirs(logfile_folder, exist_ok=True)
logfile_name = self.config.get('log', 'logfile_name')
if not logfile_name:
logfile_name = self.DEFAULT_LOGFILENAME
logfile = os.path.join(logfile_folder, logfile_name)
if self.config.get('log', 'rotate'):
logfile_size = self.config.get('log', 'logfile_size') << 10
max_logfiles = self.config.get('log', 'max_logfiles')
filehdlr = logging.handlers.RotatingFileHandler(
logfile,
maxBytes=logfile_size,
backupCount=max_logfiles,
encoding=locale.getpreferredencoding(do_setlocale=False))
else:
filehdlr = logging.FileHandler(
logfile,
encoding=locale.getpreferredencoding(do_setlocale=False))
filehdlr.setFormatter(fileform)
self.log.addHandler(filehdlr)
def _setup_filelog(logger, config):
fmt = '%(asctime)s %(levelname)-8s %(message)s'
datefmt = '%Y-%m-%d %H:%M:%S'
fileform = logging.Formatter(fmt, datefmt)
filelog_folder = config.get('log', 'filelog_folder')
makedirs(filelog_folder, exist_ok=True)
filelog_name = config.get('log', 'filelog_name')
filelog = os.path.join(filelog_folder, filelog_name)
if config.get('log', 'rotate'):
filelog_size = config.get('log', 'filelog_size') << 10
max_logfiles = config.get('log', 'max_logfiles')
filehdlr = logging.handlers.RotatingFileHandler(
filelog,
maxBytes=filelog_size,
backupCount=max_logfiles,
encoding=locale.getpreferredencoding(do_setlocale=False))
else:
filehdlr = logging.FileHandler(
filelog,
encoding=locale.getpreferredencoding(do_setlocale=False))
filehdlr.setFormatter(fileform)
logger.addHandler(filehdlr)
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False):
"""Downloads the content at url to download folder
:param disposition: if True and server provides content-disposition header\
the filename will be changed if needed
:return: The location where the file was saved
"""
self.checkForSameFiles()
self.checkAbort()
self.pyfile.setStatus("downloading")
download_folder = self.config['general']['download_folder']
location = save_join(download_folder, self.pyfile.package().folder)
if not exists(location):
makedirs(location, int(self.core.config["permission"]["folder"], 8))
if self.core.config["permission"]["change_dl"] and os.name != "nt":
try:
uid = getpwnam(self.config["permission"]["user"])[2]
gid = getgrnam(self.config["permission"]["group"])[2]
chown(location, uid, gid)
except Exception, e:
self.log.warning(_("Setting User and Group failed: %s") % str(e))
def conf_path(self, trans=False):
if trans:
gettext.setpaths(
[join(os.sep, "usr", "share", "pyload", "locale"), None])
translation = gettext.translation(
"setup",
join(self.path, "locale"),
languages=[self.config["general"]["language"], "en"],
fallback=True)
translation.install(True)
print _(
"Setting new configpath, current configuration will not be transferred!"
)
path = self.ask(_("Config path"), abspath(""))
try:
path = join(pypath, path)
if not exists(path):
makedirs(path)
f = open(join(pypath, "pyload", "config", "configdir"), "wb")
f.write(path)
f.close()
print _(
"Config path changed, setup will now close, please restart to go on."
)
print _("Press Enter to exit.")
raw_input()
exit()
except Exception, e:
print _("Setting config path failed: %s") % str(e)
def download(self,
url,
get={},
post={},
ref=True,
cookies=True,
disposition=False):
"""Downloads the content at url to download folder
:param disposition: if True and server provides content-disposition header\
the filename will be changed if needed
:return: The location where the file was saved
"""
self.checkForSameFiles()
self.checkAbort()
self.pyfile.setStatus("downloading")
download_folder = self.config['general']['download_folder']
location = save_join(download_folder, self.pyfile.package().folder)
if not exists(location):
makedirs(location, int(self.core.config["permission"]["folder"],
8))
if self.core.config["permission"]["change_dl"] and os.name != "nt":
try:
uid = getpwnam(self.config["permission"]["user"])[2]
gid = getgrnam(self.config["permission"]["group"])[2]
chown(location, uid, gid)
except Exception, e:
self.log.warning(
_("Setting User and Group failed: %s") % str(e))
def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False):
"""Load content at url and returns it
:param url: url as string
:param get: GET as dict
:param post: POST as dict, list or string
:param ref: Set HTTP_REFERER header
:param cookies: use saved cookies
:param just_header: if True only the header will be retrieved and returned as dict
:param decode: Whether to decode the output according to http header, should be True in most cases
:return: Loaded content
"""
if not hasattr(self, "req"): raise Exception("Plugin type does not have Request attribute.")
self.checkAbort()
res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode)
if self.core.debug:
from inspect import currentframe
frame = currentframe()
if not exists(join("tmp", self.__name__)):
makedirs(join("tmp", self.__name__))
f = open(
join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno))
, "wb")
del frame # delete the frame or it wont be cleaned
try:
tmp = res.encode("utf8")
except:
tmp = res
f.write(tmp)
f.close()
if just_header:
#parse header
header = {"code": self.req.code}
for line in res.splitlines():
line = line.strip()
if not line or ":" not in line: continue
key, none, value = line.partition(":")
key = key.lower().strip()
value = value.strip()
if key in header:
if type(header[key]) == list:
header[key].append(value)
else:
header[key] = [header[key], value]
else:
header[key] = value
res = header
return res
def load(self, url, get={}, post={}, ref=True,
cookies=True, just_header=False, decode=False):
"""
Load content at url and returns it
:param url: url as string
:param get: GET as dict
:param post: POST as dict, list or string
:param ref: Set HTTP_REFERER header
:param cookies: use saved cookies
:param just_header: if True only the header will be retrieved and returned as dict
:param decode: Whether to decode the output according to http header, should be True in most cases
:return: Loaded content
"""
if not hasattr(self, "req"):
raise Exception("Plugin type does not have Request attribute")
self.check_abort()
res = self.req.load(url, get, post, ref, cookies,
just_header, decode=decode)
if self.pyload_core.debug:
from inspect import currentframe
frame = currentframe()
dumpdir = os.path.join(self.pyload_core.cachedir,
'plugins', self.__name__)
makedirs(dumpdir, exist_ok=True)
filepath = os.path.join(dumpdir, "dump_{0}_line{1}.html".format(
frame.f_back.f_code.co_name, frame.f_back.f_lineno))
with lopen(filepath, mode='wb') as fp:
fp.write(res)
del frame # delete the frame or it wont be cleaned
if just_header:
# parse header
header = {'code': self.req.code}
for line in res.splitlines():
line = line.strip()
if not line or ":" not in line:
continue
key, none, value = line.partition(":")
key = key.lower().strip()
value = value.strip()
if key in header:
if isinstance(header[key], list):
header[key].append(value)
else:
header[key] = [header[key], value]
else:
header[key] = value
res = header
return res
def _setup_storage(self):
storage_folder = self.config.get('general', 'storage_folder')
if not storage_folder:
storage_folder = os.path.join(USERDIR, self.DEFAULT_STORAGEDIRNAME)
self.log.debug("Storage: {0}".format(storage_folder))
makedirs(storage_folder, exist_ok=True)
avail_space = format.size(availspace(storage_folder))
self.log.info(
self._("Available storage space: {0}").format(avail_space))
def _init_cache(self):
# Re-use cache
tempdir = self.__tempdir
if tempdir is None:
tempdir = self.session.get('previous', 'cache', 'path')
if tempdir is None or not os.path.isdir(tempdir):
pydir = os.path.join(TMPDIR, __namespace__)
makedirs(pydir, exist_ok=True)
tempdir = tempfile.mkdtemp(dir=pydir)
self.session.set('current', 'cache', 'path', tempdir)
self.cachedir = tempdir
def _setup_storage(self):
self.log.debug('Setup storage...')
storage_folder = self.config.get('general', 'storage_folder')
if storage_folder is None:
storage_folder = os.path.join(builtins.USERDIR,
self.DEFAULT_STORAGENAME)
self.log.info(self._('Storage: {0}'.format(storage_folder)))
makedirs(storage_folder, exist_ok=True)
avail_space = format.size(availspace(storage_folder))
self.log.info(
self._('Available storage space: {0}').format(avail_space))
def _setup_storage(self):
self.log.debug('Setup storage...')
storage_folder = self.config.get('general', 'storage_folder')
if storage_folder is None:
storage_folder = os.path.join(
builtins.USERDIR, self.DEFAULT_STORAGENAME)
self.log.info(self._('Storage: {0}'.format(storage_folder)))
makedirs(storage_folder, exist_ok=True)
avail_space = format.size(availspace(storage_folder))
self.log.info(
self._('Available storage space: {0}').format(avail_space))
def _mkdprofile(profile=None, rootdir=None):
DEFAULT_PROFILE = 'default'
if not profile:
profile = DEFAULT_PROFILE
if rootdir is None:
dirname = '.' + __namespace__ if os.name != 'nt' else __namespace__
configdir = os.path.join(DATADIR, dirname)
else:
configdir = os.path.expanduser(rootdir)
profiledir = os.path.realpath(os.path.join(configdir, profile))
makedirs(profiledir, exist_ok=True)
return profiledir
def debug_report(self, file):
dumpdir = os.path.join(self.pyload.tmpdir, 'plugins', file.pluginname)
makedirs(dumpdir, exist_ok=True)
# NOTE: Relpath to configdir
reportdir = os.path.join('crashes', 'plugins', file.pluginname)
makedirs(reportdir, exist_ok=True)
filename = 'debug-report_{0}_{1}.zip'.format(
file.pluginname, time.strftime('%d-%m-%Y_%H-%M-%S'))
filepath = os.path.join(reportdir, filename)
reports = self._gen_reports(file)
self._zip(filepath, reports, dumpdir)
self.pyload.log.info(
self._('Debug Report written to file {0}').format(filename))
def conf_path(self, trans=False):
if trans:
translation = gettext.translation("setup", join(self.path, "locale"),
languages=[self.config["general"]["language"], "en"], fallback=True)
translation.install(True)
print _("Setting new configpath, current configuration will not be transferred!")
path = self.ask(_("Config path"), abspath(""))
try:
path = join(pypath, path)
if not exists(path):
makedirs(path)
f = open(join(pypath, "pyload", "config", "configdir"), "wb")
f.write(path)
f.close()
print _("Config path changed, setup will now close, please restart to go on.")
print _("Press Enter to exit.")
raw_input()
exit()
except Exception, e:
print _("Setting config path failed: %s") % str(e)
def _setup_filelog(logger, config):
fmt = '%(asctime)s %(levelname)-8s %(message)s'
datefmt = '%Y-%m-%d %H:%M:%S'
fileform = logging.Formatter(fmt, datefmt)
filelog_folder = config.get('log', 'filelog_folder')
makedirs(filelog_folder, exist_ok=True)
filelog_name = config.get('log', 'filelog_name')
filelog = os.path.join(filelog_folder, filelog_name)
if config.get('log', 'rotate'):
filelog_size = config.get('log', 'filelog_size') << 10
max_logfiles = config.get('log', 'max_logfiles')
filehdlr = logging.handlers.RotatingFileHandler(
filelog, maxBytes=filelog_size, backupCount=max_logfiles,
encoding=locale.getpreferredencoding(do_setlocale=False))
else:
filehdlr = logging.FileHandler(
filelog, encoding=locale.getpreferredencoding(
do_setlocale=False))
filehdlr.setFormatter(fileform)
logger.addHandler(filehdlr)
def tearDownClass(cls):
name = '{0}.{1}'.format(cls.__module__, cls.__name__)
makedirs(name, exist_ok=True)
for fname in glob('debug_*'):
shutil.move(fname, os.path.join(name, fname))
def download(self, url, get=None, post=None, ref=True,
cookies=True, disposition=False):
"""Downloads the content at url to download folder.
:param disposition: if True and server provides content-disposition header
the filename will be changed if needed
:return: The location where the file was saved
"""
self.check_for_same_files()
self.check_abort()
self.file.set_status('downloading')
download_folder = self.pyload.config.get('general', 'storage_folder')
location = os.path.join(download_folder, self.file.package().folder)
if not os.path.isdir(location):
mode = self.pyload.config.get('permission', 'foldermode')
makedirs(location, mode, exist_ok=True)
if self.pyload.config.get(
'permission', 'change_fileowner') and os.name != 'nt':
try:
uid = pwd.getpwnam(self.pyload.config.get(
'permission', 'user'))[2]
gid = grp.getgrnam(self.pyload.config.get(
'permission', 'group'))[2]
os.chown(location, uid, gid)
except Exception as exc:
self.pyload.log.warning(
self._('Setting User and Group failed'))
self.pyload.log.error(exc, exc_info=self.pyload.debug)
name = self.file.name
filepath = os.path.join(location, name)
self.pyload.adm.fire('download:start', self.file, url, filepath)
# Create the class used for downloading
self.dl = self.pyload.req.get_download_request(
self.req, self.DOWNLOAD_CLASS)
try:
# TODO: hardcoded arguments
newname = self.dl.download(
url, filepath, get=get, post=post, referer=ref,
chunks=self.get_chunk_count(),
resume=self.resume_download, cookies=cookies,
disposition=disposition)
finally:
self.dl.close()
self.file.size = self.dl.size
if disposition and newname and newname != name: # triple check, just to be sure
self.pyload.log.info(
self._('{0} saved as {1}').format(name, newname))
self.file.name = newname
filepath = os.path.join(location, newname)
fs_filename = filepath
if self.pyload.config.get('permission', 'change_filemode'):
os.chmod(fs_filename, self.pyload.config.get(
'permission', 'filemode'))
if self.pyload.config.get(
'permission', 'change_fileowner') and os.name != 'nt':
try:
uid = pwd.getpwnam(
self.pyload.config.get(
'permission', 'user'))[2]
gid = grp.getgrnam(self.pyload.config.get(
'permission', 'group'))[2]
os.chown(fs_filename, uid, gid)
except Exception as exc:
self.pyload.log.warning(
self._('Setting User and Group failed'))
self.pyload.log.error(exc, exc_info=self.pyload.debug)
self.last_download = fs_filename
return self.last_download
def tearDownClass(cls):
name = '{0}.{1}'.format(cls.__module__, cls.__name__)
makedirs(name, exist_ok=True)
for fname in glob('debug_*'):
shutil.move(fname, os.path.join(name, fname))
def download(self,
url,
get=None,
post=None,
ref=True,
cookies=True,
disposition=False):
"""Downloads the content at url to download folder.
:param disposition: if True and server provides content-disposition header
the filename will be changed if needed
:return: The location where the file was saved
"""
self.check_for_same_files()
self.check_abort()
self.file.set_status('downloading')
download_folder = self.pyload.config.get('general', 'storage_folder')
location = os.path.join(download_folder, self.file.package().folder)
if not os.path.isdir(location):
mode = self.pyload.config.get('permission', 'foldermode')
makedirs(location, mode, exist_ok=True)
if self.pyload.config.get('permission',
'change_fileowner') and os.name != 'nt':
try:
uid = pwd.getpwnam(
self.pyload.config.get('permission', 'user'))[2]
gid = grp.getgrnam(
self.pyload.config.get('permission', 'group'))[2]
os.chown(location, uid, gid)
except Exception as exc:
self.pyload.log.warning(
self._('Setting User and Group failed'))
self.pyload.log.error(exc, exc_info=self.pyload.debug)
name = self.file.name
filepath = os.path.join(location, name)
self.pyload.adm.fire('download:start', self.file, url, filepath)
# Create the class used for downloading
self.dl = self.pyload.req.get_download_request(self.req,
self.DOWNLOAD_CLASS)
try:
# TODO: hardcoded arguments
newname = self.dl.download(url,
filepath,
get=get,
post=post,
referer=ref,
chunks=self.get_chunk_count(),
resume=self.resume_download,
cookies=cookies,
disposition=disposition)
finally:
self.dl.close()
self.file.size = self.dl.size
if disposition and newname and newname != name: # triple check, just to be sure
self.pyload.log.info(
self._('{0} saved as {1}').format(name, newname))
self.file.name = newname
filepath = os.path.join(location, newname)
fs_filename = filepath
if self.pyload.config.get('permission', 'change_filemode'):
os.chmod(fs_filename,
self.pyload.config.get('permission', 'filemode'))
if self.pyload.config.get('permission',
'change_fileowner') and os.name != 'nt':
try:
uid = pwd.getpwnam(self.pyload.config.get(
'permission', 'user'))[2]
gid = grp.getgrnam(
self.pyload.config.get('permission', 'group'))[2]
os.chown(fs_filename, uid, gid)
except Exception as exc:
self.pyload.log.warning(
self._('Setting User and Group failed'))
self.pyload.log.error(exc, exc_info=self.pyload.debug)
self.last_download = fs_filename
return self.last_download