def parse_domains(self, list):
regexp = re.compile(r'^(?:https?://)?(?:www\.)?(?:\w+\.)*((?:[\d.]+|[\w\-^_]{3,63}(?:\.[a-zA-Z]{2,}){1,2})(?:\:\d+)?)',
re.I | re.U)
r'^(?:https?://)?(?:www\.)?(?:\w+\.)*((?:[\d.]+|[\w\-^_]{3,63}(?:\.[a-zA-Z]{2,}){1,2})(?:\:\d+)?)'
domains = [decode(domain).strip().lower() for url in list for domain in regexp.findall(url)]
return self.replace_domains(uniqify(domains))
def add_password(self, password):
"""
Adds a password to saved list
"""
try:
self.passwords = uniqify([password] + self.passwords)
file = encode(self.get_config('passwordfile'))
with open(file, "wb") as f:
for pw in self.passwords:
f.write(pw + '\n')
except IOError, e:
self.log_error(e)
def _extract(self, pyfile, archive, password):
name = os.path.basename(archive.filename)
pyfile.setStatus("processing")
encrypted = False
try:
self.log_debug("Password: %s" % (password or "None provided"))
passwords = uniqify([password] + self.get_passwords(False)) if self.get_config('usepasswordfile') else [password]
for pw in passwords:
try:
pyfile.setCustomStatus(_("archive testing"))
pyfile.setProgress(0)
archive.verify(pw)
pyfile.setProgress(100)
except PasswordError:
if not encrypted:
self.log_info(name, _("Password protected"))
encrypted = True
except CRCError, e:
self.log_debug(name, e)
self.log_info(name, _("CRC Error"))
if not self.repair:
raise CRCError("Archive damaged")
else:
self.log_warning(name, _("Repairing..."))
pyfile.setCustomStatus(_("archive repairing"))
pyfile.setProgress(0)
repaired = archive.repair()
pyfile.setProgress(100)
if not repaired and not self.get_config('keepbroken'):
raise CRCError("Archive damaged")
else:
self.add_password(pw)
break
except ArchiveError, e:
raise ArchiveError(e)
except ArchiveError, e:
raise ArchiveError(e)
else:
self.add_password(pw)
break
pyfile.setCustomStatus(_("archive extracting"))
pyfile.setProgress(0)
if not encrypted or not self.get_config('usepasswordfile'):
self.log_debug("Extracting using password: %s" % (password or "None"))
archive.extract(password)
else:
for pw in filter(None, uniqify([password] + self.get_passwords(False))):
try:
self.log_debug("Extracting using password: %s" % pw)
archive.extract(pw)
self.add_password(pw)
break
except PasswordError:
self.log_debug("Password was wrong")
else:
raise PasswordError
pyfile.setProgress(100)
pyfile.setStatus("processing")
def get_links(self):
f = lambda url: "http://" + re.sub(r'(\w{7})s\.', r'\1.', url)
return uniqify(map(f, re.findall(self.LINK_PATTERN, self.data)))