def startExtracting(self, plugin, fid, passwords, thread):
pyfile = self.core.files.getFile(fid)
if not pyfile:
return []
pyfile.setCustomStatus(_("extracting"))
thread.addActive(pyfile) # keep this file until everything is done
try:
progress = lambda x: pyfile.setProgress(x)
success = False
if not plugin.checkArchive():
plugin.extract(progress)
success = True
else:
self.logInfo(basename(plugin.file), _("Password protected"))
self.logDebug("Passwords: %s" % str(passwords))
pwlist = copy(self.getPasswords())
#remove already supplied pws from list (only local)
for pw in passwords:
if pw in pwlist:
pwlist.remove(pw)
for pw in passwords + pwlist:
try:
self.logDebug("Try password: %s" % pw)
if plugin.checkPassword(pw):
plugin.extract(progress, pw)
self.addPassword(pw)
success = True
break
except WrongPassword:
self.logDebug("Password was wrong")
if not success:
self.logError(basename(plugin.file), _("Wrong password"))
return []
if self.core.debug:
self.logDebug("Would delete: %s" % ", ".join(plugin.getDeleteFiles()))
if self.getConfig("deletearchive"):
files = plugin.getDeleteFiles()
self.logInfo(_("Deleting %s files") % len(files))
for f in files:
if exists(f):
remove(f)
else:
self.logDebug("%s does not exists" % f)
self.logInfo(basename(plugin.file), _("Extracting finished"))
self.manager.dispatchEvent("unrarFinished", plugin.out, plugin.file)
return plugin.getExtractedFiles()
except ArchiveError, e:
self.logError(basename(plugin.file), _("Archive Error"), str(e))
def setUp(self):
PluginTester.setUp(self)
for f in self.files:
if exists(save_join(DL_DIR, f)): remove(save_join(DL_DIR, f))
# folder for reports
report = join("tmp", self.__class__.__name__)
if exists(report):
for f in listdir(report):
remove(join(report, f))
def setUp(self):
PluginTester.setUp(self)
for f in self.files:
if exists(save_join(DL_DIR, f)): remove(save_join(DL_DIR, f))
# folder for reports
report = join("tmp", self.__class__.__name__)
if exists(report):
for f in listdir(report):
remove(join(report, f))
def _decrypt(self, urls):
""" Internal method to select decrypting method
:param urls: List of urls/content
:return:
"""
cls = self.__class__
# separate local and remote files
content, urls = self.getLocalContent(urls)
if has_method(cls, "decryptURLs"):
self.setup()
result = to_list(self.decryptURLs(urls))
elif has_method(cls, "decryptURL"):
result = []
for url in urls:
self.setup()
result.extend(to_list(self.decryptURL(url)))
elif has_method(cls, "decrypt"):
self.logDebug("Deprecated .decrypt() method in Crypter plugin")
result = []
for url in urls:
self.pyfile = PyFileMockup(url, self.package)
self.setup()
self.decrypt(self.pyfile)
result.extend(self.convertPackages())
else:
if not has_method(cls, "decryptFile") or urls:
self.logDebug(
"No suited decrypting method was overwritten in plugin")
result = []
if has_method(cls, "decryptFile"):
for f, c in content:
self.setup()
result.extend(to_list(self.decryptFile(c)))
try:
if f.startswith("tmp_"): remove(f)
except:
pass
return result
def _decrypt(self, urls):
""" Internal method to select decrypting method
:param urls: List of urls/content
:return:
"""
cls = self.__class__
# separate local and remote files
content, urls = self.getLocalContent(urls)
if has_method(cls, "decryptURLs"):
self.setup()
result = to_list(self.decryptURLs(urls))
elif has_method(cls, "decryptURL"):
result = []
for url in urls:
self.setup()
result.extend(to_list(self.decryptURL(url)))
elif has_method(cls, "decrypt"):
self.logDebug("Deprecated .decrypt() method in Crypter plugin")
result = []
for url in urls:
self.pyfile = PyFileMockup(url, self.package)
self.setup()
self.decrypt(self.pyfile)
result.extend(self.convertPackages())
else:
if not has_method(cls, "decryptFile") or urls:
self.logDebug("No suited decrypting method was overwritten in plugin")
result = []
if has_method(cls, "decryptFile"):
for f, c in content:
self.setup()
result.extend(to_list(self.decryptFile(c)))
try:
if f.startswith("tmp_"): remove(f)
except :
pass
return result
def checkDownload(self,
rules,
api_size=0,
max_size=50000,
delete=True,
read_size=0):
""" checks the content of the last downloaded file, re match is saved to `lastCheck`
:param rules: dict with names and rules to match (compiled regexp or strings)
:param api_size: expected file size
:param max_size: if the file is larger then it wont be checked
:param delete: delete if matched
:param read_size: amount of bytes to read from files larger then max_size
:return: dictionary key of the first rule that matched
"""
lastDownload = fs_encode(self.lastDownload)
if not exists(lastDownload): return None
size = stat(lastDownload)
size = size.st_size
if api_size and api_size <= size: return None
elif size > max_size and not read_size: return None
self.log.debug("Download Check triggered")
f = open(lastDownload, "rb")
content = f.read(read_size if read_size else -1)
f.close()
#produces encoding errors, better log to other file in the future?
#self.log.debug("Content: %s" % content)
for name, rule in rules.iteritems():
if type(rule) in (str, unicode):
if rule in content:
if delete:
remove(lastDownload)
return name
elif hasattr(rule, "search"):
m = rule.search(content)
if m:
if delete:
remove(lastDownload)
self.lastCheck = m
return name
def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0):
""" checks the content of the last downloaded file, re match is saved to `lastCheck`
:param rules: dict with names and rules to match (compiled regexp or strings)
:param api_size: expected file size
:param max_size: if the file is larger then it wont be checked
:param delete: delete if matched
:param read_size: amount of bytes to read from files larger then max_size
:return: dictionary key of the first rule that matched
"""
lastDownload = fs_encode(self.lastDownload)
if not exists(lastDownload): return None
size = stat(lastDownload)
size = size.st_size
if api_size and api_size <= size: return None
elif size > max_size and not read_size: return None
self.log.debug("Download Check triggered")
f = open(lastDownload, "rb")
content = f.read(read_size if read_size else -1)
f.close()
#produces encoding errors, better log to other file in the future?
#self.log.debug("Content: %s" % content)
for name, rule in rules.iteritems():
if type(rule) in (str, unicode):
if rule in content:
if delete:
remove(lastDownload)
return name
elif hasattr(rule, "search"):
m = rule.search(content)
if m:
if delete:
remove(lastDownload)
self.lastCheck = m
return name
def init(self):
"""main loop, which executes commands"""
version = self._checkVersion()
self.conn = sqlite3.connect(self.DB_FILE)
chmod(self.DB_FILE, 0600)
self.c = self.conn.cursor()
if version is not None and version < DB_VERSION:
success = self._convertDB(version)
# delete database
if not success:
self.c.close()
self.conn.close()
try:
self.manager.core.log.warning(
_("File database was deleted due to incompatible version."
))
except:
print "File database was deleted due to incompatible version."
remove(self.VERSION_FILE)
move(self.DB_FILE, self.DB_FILE + ".backup")
f = open(self.VERSION_FILE, "wb")
f.write(str(DB_VERSION))
f.close()
self.conn = sqlite3.connect(self.DB_FILE)
chmod(self.DB_FILE, 0600)
self.c = self.conn.cursor()
self._createTables()
self.conn.commit()
def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg',
result_type='textual'):
""" Loads a captcha and decrypts it with ocr, plugin, user input
:param url: url of captcha image
:param get: get part for request
:param post: post part for request
:param cookies: True if cookies should be enabled
:param forceUser: if True, ocr is not used
:param imgtype: Type of the Image
:param result_type: 'textual' if text is written on the captcha\
or 'positional' for captcha where the user have to click\
on a specific region on the captcha
:return: result of decrypting
"""
img = self.load(url, get=get, post=post, cookies=cookies)
id = ("%.2f" % time())[-6:].replace(".", "")
temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb")
temp_file.write(img)
temp_file.close()
name = "%sOCR" % self.__name__
has_plugin = name in self.core.pluginManager.getPlugins("internal")
if self.core.captcha:
OCR = self.core.pluginManager.loadClass("internal", name)
else:
OCR = None
if OCR and not forceUser:
sleep(randint(3000, 5000) / 1000.0)
self.checkAbort()
ocr = OCR()
result = ocr.get_captcha(temp_file.name)
else:
task = self.im.createCaptchaTask(img, imgtype, temp_file.name, self.__name__, result_type)
self.task = task
while task.isWaiting():
if self.abort():
self.im.removeTask(task)
raise Abort()
sleep(1)
#TODO task handling
self.im.removeTask(task)
if task.error and has_plugin: #ignore default error message since the user could use OCR
self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting"))
elif task.error:
self.fail(task.error)
elif not task.result:
self.fail(_("No captcha result obtained in appropriate time."))
result = task.result
self.log.debug("Received captcha result: %s" % str(result))
if not self.core.debug:
try:
remove(temp_file.name)
except:
pass
return result
def decryptCaptcha(self,
url,
get={},
post={},
cookies=False,
forceUser=False,
imgtype='jpg',
result_type='textual'):
""" Loads a captcha and decrypts it with ocr, plugin, user input
:param url: url of captcha image
:param get: get part for request
:param post: post part for request
:param cookies: True if cookies should be enabled
:param forceUser: if True, ocr is not used
:param imgtype: Type of the Image
:param result_type: 'textual' if text is written on the captcha\
or 'positional' for captcha where the user have to click\
on a specific region on the captcha
:return: result of decrypting
"""
img = self.load(url, get=get, post=post, cookies=cookies)
id = ("%.2f" % time())[-6:].replace(".", "")
temp_file = open(
join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)),
"wb")
temp_file.write(img)
temp_file.close()
name = "%sOCR" % self.__name__
has_plugin = name in self.core.pluginManager.getPlugins("internal")
if self.core.captcha:
OCR = self.core.pluginManager.loadClass("internal", name)
else:
OCR = None
if OCR and not forceUser:
sleep(randint(3000, 5000) / 1000.0)
self.checkAbort()
ocr = OCR()
result = ocr.get_captcha(temp_file.name)
else:
task = self.im.newCaptchaTask(img, imgtype, temp_file.name,
result_type)
self.task = task
self.im.handleTask(task)
while task.isWaiting():
if self.abort():
self.im.removeTask(task)
raise Abort()
sleep(1)
#TODO task handling
self.im.removeTask(task)
if task.error and has_plugin: #ignore default error message since the user could use OCR
self.fail(
_("Pil and tesseract not installed and no Client connected for captcha decrypting"
))
elif task.error:
self.fail(task.error)
elif not task.result:
self.fail(
_("No captcha result obtained in appropriate time by any of the plugins."
))
result = task.result
self.log.debug("Received captcha result: %s" % str(result))
if not self.core.debug:
try:
remove(temp_file.name)
except:
pass
return result
