def gen_request(**kwargs):
__start()
try:
openssl = __edr.config('Dirs')['openssl_path']
if not openssl:
return
except KeyError:
return
con, cur = __edr.DBConnect()
request_date = datetime.strftime(datetime.now(), "%Y-%m-%dT%H:%M:%S%z")
# else:
# request_date = "2012-01-01T01:01:01.000+06:00"
cur.close()
con.close()
request_text = """<?xml version="1.0" encoding="windows-1251"?>
<request>
<requestTime>%s</requestTime>
<operatorName>Общество с ограниченной ответственностью «ВиЭйчДжи»</operatorName>
<inn>7202217753</inn>
<ogrn>1117232016076</ogrn>
<email>[email protected]</email>
</request>
""" % request_date
# datetime.strftime(datetime.now() - timedelta(days=7), "%Y-%m-%dT%H:%M:%S%z")
request_path = __edr.config('Dirs')['xml_file_name']
request_sig_path = __edr.config('Dirs')['sig_file_name']
pem_file = __edr.config('Dirs')['pem_file_name']
request_file = open(request_path, 'w')
request_file.write(request_text)
request_file.close()
os.system("%(openssl)s smime -sign -in %(zapros)s -out %(zapros_sig)s -binary -signer %(pem)s -outform DER -nodetach" %
{"zapros": request_path, "zapros_sig": request_sig_path, "pem": pem_file, 'openssl': openssl})
def zabbix_check_status_write(status):
"""Пишем статус проверки в файл, для zabbix"""
if __edr.config('Dirs')['zb_check_file']:
zb_check_status_file = __edr.config('Dirs')['zb_check_file']
zb_file = open(zb_check_status_file, "w")
if status:
zb_file.write("1\n")
__edr.printt("Writing to zb_check_file 1")
__edr.LogWrite("Writing to zb_check_file 1", "zb_check")
else:
zb_file.write("0\n")
__edr.printt("Writing to zb_check_file 0")
__edr.LogWrite("Writing to zb_check_file 0", "zb_check")
zb_file.close()
def main(**kwargs):
if __edr.str2bool(__edr.config('Main')['nginx']):
__start()
try:
with PidFile("zapretnginx.py.pid"):
__genereate(**kwargs)
except PidFileError:
__edr.printt("Идёт обновление базы, выполненние невозможно.")
__edr.LogWrite("Идёт обновление базы, выполненние невозможно.")
def __genereate():
"""
Создаём файл настроек для bind
:return:
"""
__edr.LogWrite("Genereate bind file")
skip_domain = ['youtube.com', 'www.youtube.com']
bind_file_path = __edr.config('Dirs')['bind_file']
bind_file = open(bind_file_path, 'w')
cur.execute("SELECT domain FROM edrdata WHERE disabled=0 GROUP BY domain;")
data = cur.fetchall()
for rec in data:
edr_url = rec[0].strip()
if (not edr_url.lower() in skip_domain) and (edr_url.strip()[-1:].isalpha()):
data = ('zone "%s" { type master; file "%s"; allow-query { any; }; };\n' % (
edr_url, __edr.config('Dirs')['bind_block_file']))
bind_file.write(data)
bind_file.close()
con.close()
def __gen_ipfile():
if __edr.str2bool(__edr.config('Main')['export_ip_file']):
white_list = __edr.config('Main')['white_list'].split(';')
con, cur = __edr.DBConnect()
ipfile = open(__edr.config('Dirs')['path_ip_file'] + ".tmp", 'w')
__edr.printt("Write ip's to file")
__edr.LogWrite("Write ip's to file")
if __edr.str2bool(__edr.config('Main')['export_clear_ip']):
cur.execute(
"SELECT ip FROM edrdata WHERE disabled=0 and domain='ip' GROUP BY ip;"
)
data = cur.fetchall()
cur.execute(
'SELECT ip FROM edrdata WHERE disabled=0 and domain rlike "^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$" GROUP BY ip;'
)
data += cur.fetchall()
else:
cur.execute("SELECT ip FROM edrdata WHERE disabled=0 GROUP BY ip;")
data = cur.fetchall()
cur.close()
con.close()
__edr.printt(data)
for ip in data:
for i in literal_eval(ip[0]):
if i not in white_list:
ipfile.write("%s\n" % i)
for ip in blacklist():
ipfile.write("%s\n" % ip)
ipfile.close()
copyfile(
__edr.config('Dirs')['path_ip_file'] + ".tmp",
__edr.config('Dirs')['path_ip_file'])
__edr.LogWrite("Write ip's to file done")
def __domain2ip(domain):
try:
dns_resolver = dns.resolver.Resolver()
dns_resolver.nameservers = [
__edr.config('Main')['dns_serv'], '8.8.8.8'
]
ips = dns_resolver.query(domain, 'A', raise_on_no_answer=False)
if len(ips) > 0:
return set(ip.to_text() for ip in ips)
else:
return False
except dns.exception.DNSException:
return False
def checksite(url):
result = {'notblocked': [], 'errors': [], 'blocked': []}
if url:
if url[:4] != "http":
url = "http://%s" % url
try:
answer = urllib2.urlopen(
url, timeout=int(__edr.config('Main')['check_timeout']))
tmpanswer = answer.read()
if max(word in tmpanswer
for word in __edr.config('Main')['find_words'].split("|")):
result['blocked'] = url
else:
result['notblocked'] = url
except urllib2.URLError:
result['errors'] = url
except socket.timeout:
result['errors'] = url
__edr.LogWrite("""===Blocked result: %(blocked)s
!!!Not blocked: %(notblocked)s
...With errors: %(errors)s\n""" % result,
type="zb_check")
return result
def checkblockedsites():
"""Возвращает 1, если есть не заблокированные сайты. Используется для zabbix."""
__edr.LogWrite("Start check urls", "zb_check")
f = urllib2.urlopen('http://api.antizapret.info/all.php?type=csv')
reader = csv.reader(f, delimiter=';')
result = {'notblocked': [], 'errors': [], 'blocked': []}
urls = []
count = 0
max_count = int(__edr.config('Main')['max_url_check'])
for row in reader:
if row[0] == datetime.now().strftime("%Y-%m-%d"):
continue
if max_count <= count:
break
url = row[1] or row[2]
if url:
if url[:4] != "http":
url = "http://%s" % url
urls.append(url)
count += 1
pool = ThreadPool(int(__edr.config('Main')['threads']))
results = pool.map(checksite, urls)
for i in results:
for key in result.keys():
if i[key]:
result[key].append(i[key])
__edr.printt("===\nBlocked result: %s\n" % result['blocked'])
__edr.printt("===\nNot blocked result: %s\n" % result['notblocked'])
__edr.printt("===\nWith errors: %s\n" % result['errors'])
__edr.LogWrite("===\nBlocked result: %s\n" % result['blocked'], "zb_check")
__edr.LogWrite("===\nNot blocked result: %s\n" % result['notblocked'],
"zb_check")
__edr.LogWrite("===\nWith errors: %s\n" % result['errors'], "zb_check")
pool.close()
pool.join()
zabbix_check_status_write(int(bool(result)))
return int(bool(result))
def __genereate(**kwargs):
"""
Создаём файл настроек для nginx
:return:
"""
con, cur = __edr.DBConnect()
__edr.LogWrite("Genereate nginx file")
cur.execute("SELECT url FROM edrdata WHERE disabled=0 GROUP BY domain;")
data = cur.fetchall()
domains = sorted(
set([__edr.idnaconv(urlparse(url[0]).netloc) for url in data]))
con.close()
# pool = ThreadPool(int(__edr.config('Main')['threads']))
# result = pool.map(__domainparse, domains)
result = map(__domainparse, domains)
write_to_file("\n".join(result))
nginx_conf_file_path = __edr.config('Dirs')['nginx_conf_file']
copyfile(nginx_conf_file_path + ".tmp", nginx_conf_file_path)
__edr.LogWrite("Genereate nginx file done")
def __gen_ipfile():
ipfile = open(__edr.config('Dirs')['path_ip_file'] + "_full.tmp", 'w')
con, cur = __edr.DBConnect()
white_list = __edr.config('Main')['white_list'].split(';')
if __edr.str2bool(__edr.config('Main')['export_ip_file']):
__edr.printt("Write ip's to file")
__edr.LogWrite("Write ip's to file")
cur.execute("SELECT ip FROM edrdata GROUP BY ip;")
data = cur.fetchall()
for ip in data:
for ip in literal_eval(ip[0]):
if ip not in white_list:
ipfile.write("%s\n" % ip)
__edr.printt('Writed ip: {} db\n'.format(ip))
if __edr.str2bool(__edr.config('Main')['export_dns2ip_file']):
__edr.printt("Write domain names to file")
__edr.LogWrite("Write domain names to file")
cur.execute("SELECT domain FROM edrdata GROUP BY domain;")
data = cur.fetchall()
domains = sorted(
set([
__edr.idnaconv(__clean_domain_name(domain[0]))
for domain in data
]))
ips = zapret_ipfile.blacklist()
for domain in domains:
ip = __domain2ip(domain)
if ip:
ips = ips.union(ip)
for ip in ips:
if ip not in white_list:
ipfile.write("%s\n" % ip)
__edr.printt('Writed ip: {}\n'.format(ip))
ipfile.close()
copyfile(
__edr.config('Dirs')['path_ip_file'] + "_full.tmp",
__edr.config('Dirs')['path_ip_file'] + "_full")
con.close
def __start():
__edr.config()
def __domainparse(edr_domain):
# Формируем секцию server
if '\\' in edr_domain:
edr_domain = edr_domain.split('\\')[0]
white_list = __edr.config('Main')['white_list'].split(';')
if edr_domain in white_list:
return ""
con, cur = __edr.DBConnect()
cur.execute("SELECT url FROM edrdata WHERE disabled=0 and url like %s;",
('%://' + edr_domain + '/%', ))
edr_urls = cur.fetchall()
cur.execute("SELECT url FROM edrdata WHERE disabled=0 and url like %s;",
('%://' + edr_domain, ))
edr_urls += cur.fetchall()
# try:
# cur.execute("SELECT url FROM edrdata WHERE disabled=0 and url like %s;",
# ('%://' + __edr.idnaconv(edr_domain, True) + '/%',))
# edr_urls += cur.fetchall()
# cur.execute("SELECT url FROM edrdata WHERE disabled=0 and url like %s;",
# ('%://' + __edr.idnaconv(edr_domain, True),))
# edr_urls += cur.fetchall()
# except UnicodeDecodeError as e:
# print("Cannot parse %s with error %s" % (edr_domain, e))
# except UnicodeEncodeError as e:
# print("Cannot parse %s with error %s" % (edr_domain, e))
edr_ports = sorted(
set([urlparse(i[0].strip()).scheme for i in edr_urls if i[0]]))
conf_ports = ''
for edr_port in edr_ports:
if "all" in edr_ports and edr_port != "all":
continue
if edr_port == "https":
port = '443'
elif edr_port == "http":
port = '80'
else:
port = "80;\n\tlisten 443"
conf_ports += "\tlisten %(port)s;\n" % {'port': port}
conf_server = """server {
server_name %(domain)s;
""" % {
'domain': __edr.idnaconv(edr_domain)
}
conf_server += conf_ports
# Формирует location
conf_location = ""
domain_block = 0
# query = """SELECT url FROM edrdata WHERE disabled=0 and url like \'%s\';""" % \
# ('%://' + edr_domain + '/%')
# cur.execute(query)
# edr_urls = cur.fetchall()
# query = """SELECT url FROM edrdata WHERE disabled=0 and url like \'%s\';""" % \
# ('%://' + edr_domain)
# cur.execute(query)
# edr_urls += cur.fetchall()
urls_to_write = set()
for edr_url_temp in sorted(edr_urls):
edr_url = urlparse(edr_url_temp[0].strip())
if (not edr_url.path.strip()) or (edr_url.path == '/'):
urls_to_write.add('/')
domain_block = 1
break
try:
path = edr_url.path.strip()
except UnicodeError:
path = quote(edr_url.path.strip())
if max(x in ['"', '\\', ' '] for x in edr_url.path.strip()):
path = quote(edr_url.path.strip())
urls_to_write.add(path)
for url_string in sorted(urls_to_write):
conf_location += """ location "%s" {
proxy_pass %s;
}
""" % (url_string.strip(), __edr.config('URLS')['nginx_stop_url'])
if not domain_block:
conf_location += """ location / {
proxy_pass http://$host;
}
"""
# Закрываем настройки сервера
conf_end = """ resolver %(dns_serv)s;
}
""" % {
'dns_serv': __edr.config('Main')['dns_serv']
}
try:
__edr.printt("%s\n%s\n%s" % (conf_server, conf_location, conf_end))
except UnicodeEncodeError as e:
__edr.printt(e)
try:
con.close()
return "%s\n%s\n%s" % (conf_server, conf_location, conf_end)
except UnicodeEncodeError as e:
__edr.printt(e)
raise
def write_to_file(data):
nginx_conf_file_path = __edr.config('Dirs')['nginx_conf_file']
nginx_conf_file = open(nginx_conf_file_path + ".tmp", 'w')
nginx_conf_file.write("%s\n" % data)
nginx_conf_file.close()
def __start():
__edr.config()
global con
global cur
con, cur = __edr.DBConnect()
def blacklist():
f = open(__edr.config('Dirs')['path_blacklist_ips'], 'r')
return set(line for line in f)
def main():
if __edr.str2bool(__edr.config('Main')['bind']):
__start()
__genereate()
def __start():
__edr.config()
global con
global cur
con, cur = __edr.DBConnect()
def __genereate():
"""
Создаём файл настроек для nginx
:return:
"""
__edr.LogWrite("Genereate nginx file")
nginx_conf_file_path = __edr.config('Dirs')['nginx_conf_file']
nginx_conf_file = open(nginx_conf_file_path, 'w')
cur.execute("SELECT url FROM edrdata WHERE disabled=0 GROUP BY domain;")
data = cur.fetchall()
domains = sorted(set([urlparse(url[0]).netloc for url in data]))
for edr_domain in domains:
# Формируем секцию server
cur.execute("SELECT url FROM edrdata WHERE disabled=0 and url like %s;", ('%://' + edr_domain + '/%',))
edr_urls = cur.fetchall()
# edr_ports = set(['443' if urlparse(i[0]).scheme == 'https' else '80' for i in edr_urls if i[0]])
edr_ports = set([urlparse(i[0].strip()).scheme for i in edr_urls if i[0]])
for edr_port in edr_ports:
cur.execute("SELECT url FROM edrdata WHERE disabled=0 and url like %s;",
edr_port + '://' + edr_domain + '/%')
edr_urls = cur.fetchall()
conf_server = """server {
server_name %(domain)s;
listen %(port)s;
resolver %(dns_serv)s;
""" % {'domain': edr_domain, 'port': '443' if edr_port == 'https' else '80', 'dns_serv': __edr.config('URLS')['dns_serv']}
# Формирует location
conf_location = ""
domain_block = 0
url_string = "/"
for edr_url_temp in edr_urls:
edr_url = urlparse(edr_url_temp[0].strip())
# domain_block = 0 if (edr_url.path and (not edr_url.path == '/')) else 1
if (not edr_url.path) or (edr_url.path == '/'):
domain_block = 1
if (edr_url.scheme+edr_url.netloc).__len__()+3 != edr_url_temp[0].strip().__len__():
url_string = edr_url_temp[0].strip()[(edr_url.scheme+edr_url.netloc).__len__()+3:]
# if edr_url_temp[0].strip().__contains__(" ") and re.search('[А-Я]+', edr_url_temp[0]):
# url_string = quote(url_string).replace('%3D', '=') \
# .replace('%26', '&').replace('%23', '#').replace('%3F', '?')
conf_location += """ location "%s" {
proxy_pass %s;
}
""" % (url_string, __edr.config('URLS')['nginx_stop_url'])
if not domain_block:
conf_location += """ location / {
proxy_pass http://$host;
}
"""
# Закрываем настройки сервера
conf_end = """}
"""
__edr.printt(conf_server + conf_location + conf_end)
nginx_conf_file.write(conf_server + conf_location + conf_end)
# для одиночных доменов, без урлов
cur.execute("SELECT url FROM edrdata WHERE disabled=0 GROUP BY domain;")
data = cur.fetchall()
domains = sorted(set([urlparse(url[0]).netloc for url in data]))
for edr_domain in domains:
# Формируем секцию server
if not edr_domain:
continue
cur.execute("SELECT url FROM edrdata WHERE disabled=0 and url like %s;", ('%://' + edr_domain,))
edr_urls = cur.fetchall()
if not edr_urls:
continue
edr_port = urlparse(edr_urls[0][0].strip()).scheme if edr_urls[0][0] else "http"
conf_server = """server {
server_name %(domain)s;
listen %(port)s;
""" % {'domain': edr_domain, 'port': '443' if edr_port == 'https' else '80'}
# Формирует location
conf_location = """ location / {
proxy_pass %s;
}
""" % (__edr.config('URLS')['nginx_stop_url'])
# Закрываем настройки сервера
conf_end = """}
"""
__edr.printt(conf_server + conf_location + conf_end)
nginx_conf_file.write(conf_server + conf_location + conf_end)
nginx_conf_file.close()