def check_filedupe(self): """ Checks if same file was/is downloaded within same package :param starting: indicates that the current download is going to start :raises Skip: """ pack = self.pyfile.package() for pyfile in self.pyload.files.cache.values(): if pyfile is self.pyfile: continue if pyfile.name != self.pyfile.name or pyfile.package().folder != pack.folder: continue if pyfile.status in (0, 5, 7, 12): #: (finished, waiting, starting, downloading) self.skip(pyfile.pluginname) dl_folder = self.pyload.config.get("general", "download_folder") package_folder = pack.folder if self.pyload.config.get("general", "folder_per_package") else "" dl_location = fs_join(dl_folder, package_folder, self.pyfile.name) if not exists(dl_location): return pyfile = self.pyload.db.findDuplicates(self.pyfile.id, package_folder, self.pyfile.name) if pyfile: self.skip(pyfile[0]) size = os.stat(dl_location).st_size if size >= self.pyfile.size: self.skip(_("File exists"))
def check_file(self, rules, delete=False, read_size=1048576, file_size=0, size_tolerance=1024): """ Checks the content of the last downloaded file, re match is saved to `last_check` :param rules: dict with names and rules to match (compiled regexp or strings) :param delete: delete if matched :param file_size: expected file size :param size_tolerance: size check tolerance :param read_size: amount of bytes to read from files :return: dictionary key of the first rule that matched """ do_delete = False last_download = encode(self.last_download) #@TODO: Recheck in 0.4.10 if not self.last_download or not exists(last_download): self.fail(self.pyfile.error or _("No file downloaded")) try: self.check_filesize(file_size, size_tolerance) with open(last_download, "rb") as f: content = f.read(read_size) #: Produces encoding errors, better log to other file in the future? # self.log_debug("Content: %s" % content) for name, rule in rules.items(): if isinstance(rule, basestring): if rule in content: do_delete = True return name elif hasattr(rule, "search"): m = rule.search(content) if m is not None: do_delete = True self.last_check = m return name finally: if delete and do_delete: try: os.remove(last_download) except OSError, e: self.log_warning(_("Error removing `%s`") % last_download, e) else: self.log_info(_("File deleted: ") + self.last_download) self.last_download = "" #: Recheck in 0.4.10
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True, resume=None, chunks=None): """ Downloads the content at url to download folder :param url: :param get: :param post: :param ref: :param cookies: :param disposition: if True and server provides content-disposition header\ the filename will be changed if needed :return: The location where the file was saved """ self.check_status() if self.pyload.debug: self.log_debug("DOWNLOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url", "_[1]")]) dl_url = self.fixurl(url) dl_basename = parse_name(self.pyfile.name) self.pyfile.name = dl_basename self.captcha.correct() if self.pyload.config.get("download", "skip_existing"): self.check_filedupe() self.pyfile.setStatus("downloading") dl_folder = self.pyload.config.get("general", "download_folder") dl_dirname = os.path.join(dl_folder, self.pyfile.package().folder) dl_filename = os.path.join(dl_dirname, dl_basename) dl_dir = encode(dl_dirname) dl_file = encode(dl_filename) #@TODO: Move safe-filename check to HTTPDownload in 0.4.10 if not exists(dl_dir): try: os.makedirs(dl_dir) except Exception, e: self.fail(e)
def remove_plugins(self, type_plugins): """ Delete plugins from disk """ if not type_plugins: return removed = set() self.log_debug("Requested deletion of plugins: %s" % type_plugins) for type, name in type_plugins: rootplugins = os.path.join(pypath, "module", "plugins") for dir in ("userplugins", rootplugins): py_filename = fs_join(dir, type, name + ".py") pyc_filename = py_filename + "c" if type is "hook": try: self.manager.deactivateHook(name) except Exception, e: self.log_debug(e, trace=True) for filename in (py_filename, pyc_filename): if not exists(filename): continue try: os.remove(filename) except OSError, e: self.log_warning(_("Error removing `%s`") % filename, e) else: id = (type, name) removed.add(id)
def extract(self, ids, thread=None): #@TODO: Use pypack, not pid to improve method usability if not ids: return False processed = [] extracted = [] failed = [] toList = lambda string: string.replace(' ', '').replace(',', '|').replace(';', '|').split('|') destination = self.get_config('destination') subfolder = self.get_config('subfolder') fullpath = self.get_config('fullpath') overwrite = self.get_config('overwrite') priority = self.get_config('priority') recursive = self.get_config('recursive') keepbroken = self.get_config('keepbroken') extensions = [x.lstrip('.').lower() for x in toList(self.get_config('extensions'))] excludefiles = toList(self.get_config('excludefiles')) if extensions: self.log_debug("Use for extensions: %s" % "|.".join(extensions)) #: Reload from txt file self.reload_passwords() dl_folder = self.pyload.config.get("general", "download_folder") #: Iterate packages -> extractors -> targets for pid in ids: pypack = self.pyload.files.getPackage(pid) if not pypack: self.queue.remove(pid) continue self.log_info(_("Check package: %s") % pypack.name) #: Determine output folder out = fs_join(dl_folder, pypack.folder, destination, "") #: Force trailing slash if subfolder: out = fs_join(out, pypack.folder) if not exists(out): os.makedirs(out) matched = False success = True files_ids = dict((pylink['name'], ((fs_join(dl_folder, pypack.folder, pylink['name'])), pylink['id'], out)) for pylink \ in sorted(pypack.getChildren().values(), key=lambda k: k['name'])).values() #: Remove duplicates #: Check as long there are unseen files while files_ids: new_files_ids = [] if extensions: files_ids = [(fname, fid, fout) for fname, fid, fout in files_ids \ if filter(lambda ext: fname.lower().endswith(ext), extensions)] for Extractor in self.extractors: targets = Extractor.get_targets(files_ids) if targets: self.log_debug("Targets for %s: %s" % (Extractor.__name__, targets)) matched = True for fname, fid, fout in targets: name = os.path.basename(fname) if not exists(fname): self.log_debug(name, "File not found") continue self.log_info(name, _("Extract to: %s") % fout) try: pyfile = self.pyload.files.getFile(fid) archive = Extractor(self, fname, fout, fullpath, overwrite, excludefiles, priority, keepbroken, fid) thread.addActive(pyfile) archive.init() try: new_files = self._extract(pyfile, archive, pypack.password) finally: pyfile.setProgress(100) thread.finishFile(pyfile) except Exception, e: self.log_error(name, e) success = False continue #: Remove processed file and related multiparts from list files_ids = [(fname, fid, fout) for fname, fid, fout in files_ids \ if fname not in archive.items()] self.log_debug("Extracted files: %s" % new_files) for file in new_files: self.set_permissions(file) for filename in new_files: file = encode(fs_join(os.path.dirname(archive.filename), filename)) if not exists(file): self.log_debug("New file %s does not exists" % filename) continue if recursive and os.path.isfile(file): new_files_ids.append((filename, fid, os.path.dirname(filename))) #: Append as new target self.manager.dispatchEvent("archive_extracted", pyfile, archive) files_ids = new_files_ids #: Also check extracted files if matched: if success: extracted.append(pid) self.manager.dispatchEvent("package_extracted", pypack) else: failed.append(pid) self.manager.dispatchEvent("package_extract_failed", pypack) self.failed.add(pid) else: self.log_info(_("No files found to extract")) if not matched or not success and subfolder: try: os.rmdir(out) except OSError: pass self.queue.remove(pid)
else: raise PasswordError pyfile.setProgress(100) pyfile.setStatus("processing") delfiles = archive.items() self.log_debug("Would delete: " + ", ".join(delfiles)) if self.get_config('delete'): self.log_info(_("Deleting %s files") % len(delfiles)) deltotrash = self.get_config('deltotrash') for f in delfiles: file = encode(f) if not exists(file): continue if not deltotrash: os.remove(file) else: try: send2trash.send2trash(file) except NameError: self.log_warning(_("Unable to move %s to trash") % os.path.basename(f), _("Send2Trash lib not found")) except Exception, e: self.log_warning(_("Unable to move %s to trash") % os.path.basename(f),
""" if self.pyfile.url.startswith("http"): self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1] content = self.load(self.pyfile.url) self.pyfile.url = fs_join(self.pyload.config.get("general", "download_folder"), self.pyfile.name) try: with open(self.pyfile.url, "wb") as f: f.write(encode(content)) except IOError, e: self.fail(e) else: self.pyfile.name = os.path.basename(self.pyfile.url) if not exists(self.pyfile.url): if exists(fs_join(pypath, self.pyfile.url)): self.pyfile.url = fs_join(pypath, self.pyfile.url) else: self.fail(_("File not exists")) else: self.data = self.pyfile.url #@NOTE: ??? def delete_tmp(self): if not self.pyfile.name.startswith("tmp_"): return try: os.remove(self.pyfile.url) except OSError, e:
def scan(self, pyfile, thread): avfile = encode(self.get_config('avfile')) avargs = encode(self.get_config('avargs').strip()) if not os.path.isfile(avfile): self.fail(_("Antivirus executable not found")) scanfolder = self.get_config('avtarget') is "folder" if scanfolder: dl_folder = self.pyload.config.get("general", "download_folder") package_folder = pyfile.package().folder if self.pyload.config.get("general", "folder_per_package") else "" target = fs_join(dl_folder, package_folder, pyfile.name) target_repr = "Folder: " + package_folder or dl_folder else: target = encode(pyfile.plugin.last_download) target_repr = "File: " + os.path.basename(pyfile.plugin.last_download) if not exists(target): return thread.addActive(pyfile) pyfile.setCustomStatus(_("virus scanning")) pyfile.setProgress(0) try: p = subprocess.Popen([avfile, avargs, target], bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = map(str.strip, p.communicate()) if out: self.log_info(target_repr, out) if err: self.log_warning(target_repr, err) if not self.get_config('ignore-err'): self.log_debug("Delete/Quarantine task aborted due scan error") return if p.returncode: action = self.get_config('action') if scanfolder: if action is "Antivirus default": self.log_warning(_("Delete/Quarantine task skipped in folder scan mode")) return pyfile.error = _("Infected file") try: if action is "Delete": if not self.get_config('deltotrash'): os.remove(file) else: try: send2trash.send2trash(file) except NameError: self.log_warning(_("Send2Trash lib not found, moving to quarantine instead")) pyfile.setCustomStatus(_("file moving")) shutil.move(file, self.get_config('quardir')) except Exception, e: self.log_warning(_("Unable to move file to trash: %s, moving to quarantine instead") % e.message) pyfile.setCustomStatus(_("file moving")) shutil.move(file, self.get_config('quardir')) else: self.log_debug("Successfully moved file to trash") elif action is "Quarantine": pyfile.setCustomStatus(_("file moving")) shutil.move(file, self.get_config('quardir')) except (IOError, shutil.Error), e: self.log_error(target_repr, action + " action failed!", e)