def check_filedupe(self):
"""
Checks if same file was/is downloaded within same package
:param starting: indicates that the current download is going to start
:raises Skip:
"""
pack = self.pyfile.package()
for pyfile in self.pyload.files.cache.values():
if pyfile is self.pyfile:
continue
if pyfile.name != self.pyfile.name or pyfile.package().folder != pack.folder:
continue
if pyfile.status in (0, 5, 7, 12): #: (finished, waiting, starting, downloading)
self.skip(pyfile.pluginname)
dl_folder = self.pyload.config.get("general", "download_folder")
package_folder = pack.folder if self.pyload.config.get("general", "folder_per_package") else ""
dl_location = fs_join(dl_folder, package_folder, self.pyfile.name)
if not exists(dl_location):
return
pyfile = self.pyload.db.findDuplicates(self.pyfile.id, package_folder, self.pyfile.name)
if pyfile:
self.skip(pyfile[0])
size = os.stat(dl_location).st_size
if size >= self.pyfile.size:
self.skip(_("File exists"))
def check_file(self, rules, delete=False, read_size=1048576, file_size=0, size_tolerance=1024):
"""
Checks the content of the last downloaded file, re match is saved to `last_check`
:param rules: dict with names and rules to match (compiled regexp or strings)
:param delete: delete if matched
:param file_size: expected file size
:param size_tolerance: size check tolerance
:param read_size: amount of bytes to read from files
:return: dictionary key of the first rule that matched
"""
do_delete = False
last_download = encode(self.last_download) #@TODO: Recheck in 0.4.10
if not self.last_download or not exists(last_download):
self.fail(self.pyfile.error or _("No file downloaded"))
try:
self.check_filesize(file_size, size_tolerance)
with open(last_download, "rb") as f:
content = f.read(read_size)
#: Produces encoding errors, better log to other file in the future?
# self.log_debug("Content: %s" % content)
for name, rule in rules.items():
if isinstance(rule, basestring):
if rule in content:
do_delete = True
return name
elif hasattr(rule, "search"):
m = rule.search(content)
if m is not None:
do_delete = True
self.last_check = m
return name
finally:
if delete and do_delete:
try:
os.remove(last_download)
except OSError, e:
self.log_warning(_("Error removing `%s`") % last_download, e)
else:
self.log_info(_("File deleted: ") + self.last_download)
self.last_download = "" #: Recheck in 0.4.10
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True, resume=None, chunks=None):
"""
Downloads the content at url to download folder
:param url:
:param get:
:param post:
:param ref:
:param cookies:
:param disposition: if True and server provides content-disposition header\
the filename will be changed if needed
:return: The location where the file was saved
"""
self.check_status()
if self.pyload.debug:
self.log_debug("DOWNLOAD URL " + url,
*["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url", "_[1]")])
dl_url = self.fixurl(url)
dl_basename = parse_name(self.pyfile.name)
self.pyfile.name = dl_basename
self.captcha.correct()
if self.pyload.config.get("download", "skip_existing"):
self.check_filedupe()
self.pyfile.setStatus("downloading")
dl_folder = self.pyload.config.get("general", "download_folder")
dl_dirname = os.path.join(dl_folder, self.pyfile.package().folder)
dl_filename = os.path.join(dl_dirname, dl_basename)
dl_dir = encode(dl_dirname)
dl_file = encode(dl_filename) #@TODO: Move safe-filename check to HTTPDownload in 0.4.10
if not exists(dl_dir):
try:
os.makedirs(dl_dir)
except Exception, e:
self.fail(e)
def remove_plugins(self, type_plugins):
"""
Delete plugins from disk
"""
if not type_plugins:
return
removed = set()
self.log_debug("Requested deletion of plugins: %s" % type_plugins)
for type, name in type_plugins:
rootplugins = os.path.join(pypath, "module", "plugins")
for dir in ("userplugins", rootplugins):
py_filename = fs_join(dir, type, name + ".py")
pyc_filename = py_filename + "c"
if type is "hook":
try:
self.manager.deactivateHook(name)
except Exception, e:
self.log_debug(e, trace=True)
for filename in (py_filename, pyc_filename):
if not exists(filename):
continue
try:
os.remove(filename)
except OSError, e:
self.log_warning(_("Error removing `%s`") % filename, e)
else:
id = (type, name)
removed.add(id)
def extract(self, ids, thread=None): #@TODO: Use pypack, not pid to improve method usability
if not ids:
return False
processed = []
extracted = []
failed = []
toList = lambda string: string.replace(' ', '').replace(',', '|').replace(';', '|').split('|')
destination = self.get_config('destination')
subfolder = self.get_config('subfolder')
fullpath = self.get_config('fullpath')
overwrite = self.get_config('overwrite')
priority = self.get_config('priority')
recursive = self.get_config('recursive')
keepbroken = self.get_config('keepbroken')
extensions = [x.lstrip('.').lower() for x in toList(self.get_config('extensions'))]
excludefiles = toList(self.get_config('excludefiles'))
if extensions:
self.log_debug("Use for extensions: %s" % "|.".join(extensions))
#: Reload from txt file
self.reload_passwords()
dl_folder = self.pyload.config.get("general", "download_folder")
#: Iterate packages -> extractors -> targets
for pid in ids:
pypack = self.pyload.files.getPackage(pid)
if not pypack:
self.queue.remove(pid)
continue
self.log_info(_("Check package: %s") % pypack.name)
#: Determine output folder
out = fs_join(dl_folder, pypack.folder, destination, "") #: Force trailing slash
if subfolder:
out = fs_join(out, pypack.folder)
if not exists(out):
os.makedirs(out)
matched = False
success = True
files_ids = dict((pylink['name'], ((fs_join(dl_folder, pypack.folder, pylink['name'])), pylink['id'], out)) for pylink \
in sorted(pypack.getChildren().values(), key=lambda k: k['name'])).values() #: Remove duplicates
#: Check as long there are unseen files
while files_ids:
new_files_ids = []
if extensions:
files_ids = [(fname, fid, fout) for fname, fid, fout in files_ids \
if filter(lambda ext: fname.lower().endswith(ext), extensions)]
for Extractor in self.extractors:
targets = Extractor.get_targets(files_ids)
if targets:
self.log_debug("Targets for %s: %s" % (Extractor.__name__, targets))
matched = True
for fname, fid, fout in targets:
name = os.path.basename(fname)
if not exists(fname):
self.log_debug(name, "File not found")
continue
self.log_info(name, _("Extract to: %s") % fout)
try:
pyfile = self.pyload.files.getFile(fid)
archive = Extractor(self,
fname,
fout,
fullpath,
overwrite,
excludefiles,
priority,
keepbroken,
fid)
thread.addActive(pyfile)
archive.init()
try:
new_files = self._extract(pyfile, archive, pypack.password)
finally:
pyfile.setProgress(100)
thread.finishFile(pyfile)
except Exception, e:
self.log_error(name, e)
success = False
continue
#: Remove processed file and related multiparts from list
files_ids = [(fname, fid, fout) for fname, fid, fout in files_ids \
if fname not in archive.items()]
self.log_debug("Extracted files: %s" % new_files)
for file in new_files:
self.set_permissions(file)
for filename in new_files:
file = encode(fs_join(os.path.dirname(archive.filename), filename))
if not exists(file):
self.log_debug("New file %s does not exists" % filename)
continue
if recursive and os.path.isfile(file):
new_files_ids.append((filename, fid, os.path.dirname(filename))) #: Append as new target
self.manager.dispatchEvent("archive_extracted", pyfile, archive)
files_ids = new_files_ids #: Also check extracted files
if matched:
if success:
extracted.append(pid)
self.manager.dispatchEvent("package_extracted", pypack)
else:
failed.append(pid)
self.manager.dispatchEvent("package_extract_failed", pypack)
self.failed.add(pid)
else:
self.log_info(_("No files found to extract"))
if not matched or not success and subfolder:
try:
os.rmdir(out)
except OSError:
pass
self.queue.remove(pid)
else:
raise PasswordError
pyfile.setProgress(100)
pyfile.setStatus("processing")
delfiles = archive.items()
self.log_debug("Would delete: " + ", ".join(delfiles))
if self.get_config('delete'):
self.log_info(_("Deleting %s files") % len(delfiles))
deltotrash = self.get_config('deltotrash')
for f in delfiles:
file = encode(f)
if not exists(file):
continue
if not deltotrash:
os.remove(file)
else:
try:
send2trash.send2trash(file)
except NameError:
self.log_warning(_("Unable to move %s to trash") % os.path.basename(f),
_("Send2Trash lib not found"))
except Exception, e:
self.log_warning(_("Unable to move %s to trash") % os.path.basename(f),
"""
if self.pyfile.url.startswith("http"):
self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1]
content = self.load(self.pyfile.url)
self.pyfile.url = fs_join(self.pyload.config.get("general", "download_folder"), self.pyfile.name)
try:
with open(self.pyfile.url, "wb") as f:
f.write(encode(content))
except IOError, e:
self.fail(e)
else:
self.pyfile.name = os.path.basename(self.pyfile.url)
if not exists(self.pyfile.url):
if exists(fs_join(pypath, self.pyfile.url)):
self.pyfile.url = fs_join(pypath, self.pyfile.url)
else:
self.fail(_("File not exists"))
else:
self.data = self.pyfile.url #@NOTE: ???
def delete_tmp(self):
if not self.pyfile.name.startswith("tmp_"):
return
try:
os.remove(self.pyfile.url)
except OSError, e:
def scan(self, pyfile, thread):
avfile = encode(self.get_config('avfile'))
avargs = encode(self.get_config('avargs').strip())
if not os.path.isfile(avfile):
self.fail(_("Antivirus executable not found"))
scanfolder = self.get_config('avtarget') is "folder"
if scanfolder:
dl_folder = self.pyload.config.get("general", "download_folder")
package_folder = pyfile.package().folder if self.pyload.config.get("general", "folder_per_package") else ""
target = fs_join(dl_folder, package_folder, pyfile.name)
target_repr = "Folder: " + package_folder or dl_folder
else:
target = encode(pyfile.plugin.last_download)
target_repr = "File: " + os.path.basename(pyfile.plugin.last_download)
if not exists(target):
return
thread.addActive(pyfile)
pyfile.setCustomStatus(_("virus scanning"))
pyfile.setProgress(0)
try:
p = subprocess.Popen([avfile, avargs, target],
bufsize=-1,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = map(str.strip, p.communicate())
if out:
self.log_info(target_repr, out)
if err:
self.log_warning(target_repr, err)
if not self.get_config('ignore-err'):
self.log_debug("Delete/Quarantine task aborted due scan error")
return
if p.returncode:
action = self.get_config('action')
if scanfolder:
if action is "Antivirus default":
self.log_warning(_("Delete/Quarantine task skipped in folder scan mode"))
return
pyfile.error = _("Infected file")
try:
if action is "Delete":
if not self.get_config('deltotrash'):
os.remove(file)
else:
try:
send2trash.send2trash(file)
except NameError:
self.log_warning(_("Send2Trash lib not found, moving to quarantine instead"))
pyfile.setCustomStatus(_("file moving"))
shutil.move(file, self.get_config('quardir'))
except Exception, e:
self.log_warning(_("Unable to move file to trash: %s, moving to quarantine instead") % e.message)
pyfile.setCustomStatus(_("file moving"))
shutil.move(file, self.get_config('quardir'))
else:
self.log_debug("Successfully moved file to trash")
elif action is "Quarantine":
pyfile.setCustomStatus(_("file moving"))
shutil.move(file, self.get_config('quardir'))
except (IOError, shutil.Error), e:
self.log_error(target_repr, action + " action failed!", e)