def makePages(self):
log.debug("makePages")
for chapter in self.chapters:
log.debug("make Pages for chapter %s" % chapter.number)
c = utils.checkChapterDir(self.basedir, self.title, chapter.number)
if c != 0 and self.action == "append" and \
self.what_chapter == "all":
continue
soup = BS(utils.get_url("%s%s%s" % (self.baseurl, chapter.url,
self.suffix),
self.lookups).read())
for image in soup.find_all('img'):
image_url = image.get('src')
if "manga-img" in image_url:
img_host = image_url.split('/', 3)[-2]
if img_host not in self.lookups:
self.lookups[img_host] = dnslookup(img_host, 'A')[0]
image_url = image_url.replace(img_host,
self.lookups[img_host])
image_ext = image_url.rsplit('.', 1)[-1]
last_page_number = 0
if len(chapter.Images):
last_page_number = int(chapter.Images[-1].number
.split('.')[0]) + 1
i = manga.Image(utils.pageNumber(image_url,
last_page_number),
image_url, img_host, image_ext)
chapter.Images.append(i)
sorted(chapter.Images, key=lambda img: img.number)
self.downloadBook()
def __send_email_(self, mid):
info = self.stor.getinfo(mid)
try:
mail_servers = dnslookup(info['rcpt'][1], 'mx')
except:
return False
mail_servers = sorted(mail_servers, key=itemgetter(0))
for _, mx in mail_servers:
s = SMTP(local_hostname = conf.smtp_hostname)
try:
ret_code, banner = s.connect(mx, 25)
except:
s.quit()
continue
if ret_code != 220:
s.quit()
continue
try:
s.starttls()
except:
if conf.smtp_email_tls_required:
s.quit()
continue
from_addr = conf.smtp_email_delim.join(info['from'])
to_addr = conf.smtp_email_delim.join(info['rcpt'])
message = self.stor.read(mid)
try:
s.sendmail(from_addr, [to_addr], message['message'])
except:
s.quit()
continue
s.quit()
return True
return False
def mx_scan(domain):
try:
for record in dnslookup(domain.domain_name, 'MX'):
# Redirects will be presented as str of the redirect.
if isinstance(record, tuple):
domain.add_mx_record(record)
except DNSError as error:
handle_error("[MX]", domain, error)
def dmarc_scan(domain):
# dmarc records are kept in TXT records for _dmarc.domain_name.
try:
dmarc_domain = '_dmarc.%s' % domain.domain_name
for record in dnslookup(dmarc_domain, 'TXT'):
record_text = record_to_str(record)
if record_text.startswith("\""):
record_text = record[1:-1]
# Ensure the record is a DMARC record. Some domains that redirect will cause an SPF record to show.
if record_text.startswith("v=DMARC1"):
domain.dmarc.append(record_text)
# Remove excess spacing
record_text = record_text.strip(" ")
# DMARC records follow a specific outline as to how they are defined - tag:value
# We can split this up into a easily manipulatable
tag_dict = {}
for options in record_text.split(";"):
if '=' not in options:
continue
tag = options.split("=")[0].strip()
value = options.split("=")[1].strip()
tag_dict[tag] = value
for tag in tag_dict:
if tag not in [
"v", "mailto", "rf", "p", "sp", "adkim", "aspf", "fo",
"pct", "ri", "rua", "ruf"
]:
logging.debug(
"\tWarning: Unknown DMARC mechanism {0}".format(tag))
domain.valid_dmarc = False
elif tag == "p":
domain.dmarc_policy = tag_dict[tag]
except DNSError as error:
handle_error("[DMARC]", domain, error)
def spf_scan(domain):
try:
# for record in resolver.query(domain.domain_name, 'TXT'):
for record in dnslookup(domain.domain_name, 'TXT'):
record_text = record_to_str(record)
if record_text.startswith("\""):
record_text = record_text[1:-1]
if not record_text.startswith("v=spf1"):
# Not an spf record, ignore it.
continue
domain.spf.append(record_text)
# From the found record grab the specific result when something doesn't match.
# Definitions of result come from https://www.ietf.org/rfc/rfc4408.txt
if record_text.endswith("-all"):
result = 'fail'
elif record_text.endswith("?all"):
result = "neutral"
elif record_text.endswith("~all"):
result = "softfail"
elif record_text.endswith("all") or record_text.endswith("+all"):
result = "pass"
else:
result = "neutral"
try:
query = spf.query("127.0.0.1",
"email_wizard@" + domain.domain_name,
domain.domain_name,
strict=2)
response = query.check()
except spf.AmbiguityWarning as error:
logging.debug("\t" + error.msg)
domain.syntax_errors.append(error.msg)
continue
if response[0] == 'temperror':
logging.debug(response[2])
elif response[0] == 'permerror':
logging.debug("\t" + response[2])
domain.syntax_errors.append(response[2])
elif response[0] == 'ambiguous':
logging.debug("\t" + response[2])
domain.syntax_errors.append(response[2])
elif response[0] == result:
# Everything checks out the SPF syntax seems valid.
domain.valid_spf = True
continue
else:
domain.valid_spf = False
logging.debug(
"\tResult Differs: Expected [{0}] - Actual [{1}]".format(
result, response[0]))
domain.errors.append(
"Result Differs: Expected [{0}] - Actual [{1}]".format(
result, response[0]))
except DNSError as error:
handle_error("[SPF]", domain, error)
def addLookup(self, url):
fqdn = url.split('/')[2]
if fqdn not in self.lookups:
self.lookups[fqdn] = dnslookup(fqdn, 'A')[0]