def main():
try:
with PidFile("zapret_ipfile.py.pid"):
__start()
__gen_ipfile()
except PidFileError:
__edr.printt("Уже запущено обновление.")
def __gen_ipfile():
if __edr.str2bool(__edr.config('Main')['export_ip_file']):
white_list = __edr.config('Main')['white_list'].split(';')
con, cur = __edr.DBConnect()
ipfile = open(__edr.config('Dirs')['path_ip_file'] + ".tmp", 'w')
__edr.printt("Write ip's to file")
__edr.LogWrite("Write ip's to file")
if __edr.str2bool(__edr.config('Main')['export_clear_ip']):
cur.execute(
"SELECT ip FROM edrdata WHERE disabled=0 and domain='ip' GROUP BY ip;"
)
data = cur.fetchall()
cur.execute(
'SELECT ip FROM edrdata WHERE disabled=0 and domain rlike "^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$" GROUP BY ip;'
)
data += cur.fetchall()
else:
cur.execute("SELECT ip FROM edrdata WHERE disabled=0 GROUP BY ip;")
data = cur.fetchall()
cur.close()
con.close()
__edr.printt(data)
for ip in data:
for i in literal_eval(ip[0]):
if i not in white_list:
ipfile.write("%s\n" % i)
for ip in blacklist():
ipfile.write("%s\n" % ip)
ipfile.close()
copyfile(
__edr.config('Dirs')['path_ip_file'] + ".tmp",
__edr.config('Dirs')['path_ip_file'])
__edr.LogWrite("Write ip's to file done")
def main():
try:
with PidFile("zapretdelete_duple.py.pid"):
__start()
__genereate()
except PidFileError:
__edr.printt("Уже запущено обновление.")
def __genereate():
"""
Ну и ужас творится в списках, куча двойных записей, неправильных ссылок. И так как это всё потом
передавать в настройки, нужно убрать все косяки из базы.
:return:
"""
__edr.LogWrite('Remove duplicates')
cur.execute("SELECT url FROM edrdata WHERE disabled=0;")
data = cur.fetchall()
__edr.LogWrite('Remove duplicates. Loop.')
for rec in data:
edr_url = rec[0].strip()
#edr_url2 = edr_url+"/"
cur.execute("""SELECT url FROM edrdata WHERE disabled=0 and url=%(edr_url2)s;""", {"edr_url2":edr_url[:-1]})
data2 = cur.fetchall()
for rec2 in data2:
__edr.printt("first : %s" % rec)
__edr.printt("second: %s" % rec2)
cur.execute("""DELETE FROM edrdata WHERE url = %(edr_url)s;""", {"edr_url":edr_url[:-1]})
con.commit()
__edr.LogWrite('Remove duplicates: End Loop. Start simple delete.')
cur.execute("DELETE e1 FROM edrdata e1, edrdata e2 WHERE e1.url = e2.url AND e1.id > e2.id;")
con.commit()
#__edr.LogWrite('Remove duplicates: Delete strange url.')
#cur.execute('DELETE FROM edrdata WHERE url like "%?";')
#con.commit()
#cur.execute('DELETE FROM edrdata WHERE url like "%#";')
#con.commit()
con.close()
def __genereate():
"""
Ну и ужас творится в списках, куча двойных записей, неправильных ссылок. И так как это всё потом
передавать в настройки, нужно убрать все косяки из базы.
:return:
"""
__edr.LogWrite('Remove duplicates')
cur.execute("SELECT url FROM edrdata WHERE disabled=0;")
data = cur.fetchall()
__edr.LogWrite('Remove duplicates. Loop.')
for rec in data:
edr_url = rec[0].strip()
edr_url2 = edr_url+"/"
cur.execute("SELECT url FROM edrdata WHERE disabled=0 and url=%s", edr_url2)
data2 = cur.fetchall()
for rec2 in data2:
__edr.printt("first : %s" % rec)
__edr.printt("second: %s" % rec2)
cur.execute("DELETE FROM edrdata WHERE url = %s;\n", edr_url)
con.commit()
__edr.LogWrite('Remove duplicates: End Loop. Start simple delete.')
cur.execute("DELETE e1 FROM edrdata e1, edrdata e2 WHERE e1.url = e2.url AND e1.id > e2.id;")
con.commit()
__edr.LogWrite('Remove duplicates: Delete strange url.')
cur.execute('DELETE FROM edrdata WHERE url like "%?";')
con.commit()
cur.execute('DELETE FROM edrdata WHERE url like "%#";')
con.commit()
con.close()
def main(**kwargs):
if __edr.str2bool(__edr.config('Main')['nginx']):
__start()
try:
with PidFile("zapretnginx.py.pid"):
__genereate(**kwargs)
except PidFileError:
__edr.printt("Идёт обновление базы, выполненние невозможно.")
__edr.LogWrite("Идёт обновление базы, выполненние невозможно.")
def request2db(data, **kwargs):
__start()
con, cur = __edr.DBConnect()
__edr.printt("INSERT requests SET time=%(time)s, data=%(data)s, code=%(code)s;" %
{'time': datetime.strftime(datetime.now(), "%Y-%m-%d %H:%M:%S%z"),
'data': data, 'code': kwargs.get('code', "")})
cur.execute("INSERT requests SET time=%(time)s, data=%(data)s, code=%(code)s;",
{'time': datetime.strftime(datetime.now(), "%Y-%m-%d %H:%M:%S%z"),
'data': data, 'code': kwargs.get('code', "")})
con.commit()
cur.close()
con.close()
def zabbix_check_status_write(status):
"""Пишем статус проверки в файл, для zabbix"""
if __edr.config('Dirs')['zb_check_file']:
zb_check_status_file = __edr.config('Dirs')['zb_check_file']
zb_file = open(zb_check_status_file, "w")
if status:
zb_file.write("1\n")
__edr.printt("Writing to zb_check_file 1")
__edr.LogWrite("Writing to zb_check_file 1", "zb_check")
else:
zb_file.write("0\n")
__edr.printt("Writing to zb_check_file 0")
__edr.LogWrite("Writing to zb_check_file 0", "zb_check")
zb_file.close()
def __gen_ipfile():
ipfile = open(__edr.config('Dirs')['path_ip_file'] + "_full.tmp", 'w')
con, cur = __edr.DBConnect()
white_list = __edr.config('Main')['white_list'].split(';')
if __edr.str2bool(__edr.config('Main')['export_ip_file']):
__edr.printt("Write ip's to file")
__edr.LogWrite("Write ip's to file")
cur.execute("SELECT ip FROM edrdata GROUP BY ip;")
data = cur.fetchall()
for ip in data:
for ip in literal_eval(ip[0]):
if ip not in white_list:
ipfile.write("%s\n" % ip)
__edr.printt('Writed ip: {} db\n'.format(ip))
if __edr.str2bool(__edr.config('Main')['export_dns2ip_file']):
__edr.printt("Write domain names to file")
__edr.LogWrite("Write domain names to file")
cur.execute("SELECT domain FROM edrdata GROUP BY domain;")
data = cur.fetchall()
domains = sorted(
set([
__edr.idnaconv(__clean_domain_name(domain[0]))
for domain in data
]))
ips = zapret_ipfile.blacklist()
for domain in domains:
ip = __domain2ip(domain)
if ip:
ips = ips.union(ip)
for ip in ips:
if ip not in white_list:
ipfile.write("%s\n" % ip)
__edr.printt('Writed ip: {}\n'.format(ip))
ipfile.close()
copyfile(
__edr.config('Dirs')['path_ip_file'] + "_full.tmp",
__edr.config('Dirs')['path_ip_file'] + "_full")
con.close
def checkblockedsites():
"""Возвращает 1, если есть не заблокированные сайты. Используется для zabbix."""
__edr.LogWrite("Start check urls", "zb_check")
f = urllib2.urlopen('http://api.antizapret.info/all.php?type=csv')
reader = csv.reader(f, delimiter=';')
result = {'notblocked': [], 'errors': [], 'blocked': []}
urls = []
count = 0
max_count = int(__edr.config('Main')['max_url_check'])
for row in reader:
if row[0] == datetime.now().strftime("%Y-%m-%d"):
continue
if max_count <= count:
break
url = row[1] or row[2]
if url:
if url[:4] != "http":
url = "http://%s" % url
urls.append(url)
count += 1
pool = ThreadPool(int(__edr.config('Main')['threads']))
results = pool.map(checksite, urls)
for i in results:
for key in result.keys():
if i[key]:
result[key].append(i[key])
__edr.printt("===\nBlocked result: %s\n" % result['blocked'])
__edr.printt("===\nNot blocked result: %s\n" % result['notblocked'])
__edr.printt("===\nWith errors: %s\n" % result['errors'])
__edr.LogWrite("===\nBlocked result: %s\n" % result['blocked'], "zb_check")
__edr.LogWrite("===\nNot blocked result: %s\n" % result['notblocked'],
"zb_check")
__edr.LogWrite("===\nWith errors: %s\n" % result['errors'], "zb_check")
pool.close()
pool.join()
zabbix_check_status_write(int(bool(result)))
return int(bool(result))
def __domainparse(edr_domain):
# Формируем секцию server
if '\\' in edr_domain:
edr_domain = edr_domain.split('\\')[0]
white_list = __edr.config('Main')['white_list'].split(';')
if edr_domain in white_list:
return ""
con, cur = __edr.DBConnect()
cur.execute("SELECT url FROM edrdata WHERE disabled=0 and url like %s;",
('%://' + edr_domain + '/%', ))
edr_urls = cur.fetchall()
cur.execute("SELECT url FROM edrdata WHERE disabled=0 and url like %s;",
('%://' + edr_domain, ))
edr_urls += cur.fetchall()
# try:
# cur.execute("SELECT url FROM edrdata WHERE disabled=0 and url like %s;",
# ('%://' + __edr.idnaconv(edr_domain, True) + '/%',))
# edr_urls += cur.fetchall()
# cur.execute("SELECT url FROM edrdata WHERE disabled=0 and url like %s;",
# ('%://' + __edr.idnaconv(edr_domain, True),))
# edr_urls += cur.fetchall()
# except UnicodeDecodeError as e:
# print("Cannot parse %s with error %s" % (edr_domain, e))
# except UnicodeEncodeError as e:
# print("Cannot parse %s with error %s" % (edr_domain, e))
edr_ports = sorted(
set([urlparse(i[0].strip()).scheme for i in edr_urls if i[0]]))
conf_ports = ''
for edr_port in edr_ports:
if "all" in edr_ports and edr_port != "all":
continue
if edr_port == "https":
port = '443'
elif edr_port == "http":
port = '80'
else:
port = "80;\n\tlisten 443"
conf_ports += "\tlisten %(port)s;\n" % {'port': port}
conf_server = """server {
server_name %(domain)s;
""" % {
'domain': __edr.idnaconv(edr_domain)
}
conf_server += conf_ports
# Формирует location
conf_location = ""
domain_block = 0
# query = """SELECT url FROM edrdata WHERE disabled=0 and url like \'%s\';""" % \
# ('%://' + edr_domain + '/%')
# cur.execute(query)
# edr_urls = cur.fetchall()
# query = """SELECT url FROM edrdata WHERE disabled=0 and url like \'%s\';""" % \
# ('%://' + edr_domain)
# cur.execute(query)
# edr_urls += cur.fetchall()
urls_to_write = set()
for edr_url_temp in sorted(edr_urls):
edr_url = urlparse(edr_url_temp[0].strip())
if (not edr_url.path.strip()) or (edr_url.path == '/'):
urls_to_write.add('/')
domain_block = 1
break
try:
path = edr_url.path.strip()
except UnicodeError:
path = quote(edr_url.path.strip())
if max(x in ['"', '\\', ' '] for x in edr_url.path.strip()):
path = quote(edr_url.path.strip())
urls_to_write.add(path)
for url_string in sorted(urls_to_write):
conf_location += """ location "%s" {
proxy_pass %s;
}
""" % (url_string.strip(), __edr.config('URLS')['nginx_stop_url'])
if not domain_block:
conf_location += """ location / {
proxy_pass http://$host;
}
"""
# Закрываем настройки сервера
conf_end = """ resolver %(dns_serv)s;
}
""" % {
'dns_serv': __edr.config('Main')['dns_serv']
}
try:
__edr.printt("%s\n%s\n%s" % (conf_server, conf_location, conf_end))
except UnicodeEncodeError as e:
__edr.printt(e)
try:
con.close()
return "%s\n%s\n%s" % (conf_server, conf_location, conf_end)
except UnicodeEncodeError as e:
__edr.printt(e)
raise
def __genereate():
"""
Создаём файл настроек для nginx
:return:
"""
__edr.LogWrite("Genereate nginx file")
nginx_conf_file_path = __edr.config('Dirs')['nginx_conf_file']
nginx_conf_file = open(nginx_conf_file_path, 'w')
cur.execute("SELECT url FROM edrdata WHERE disabled=0 GROUP BY domain;")
data = cur.fetchall()
domains = sorted(set([urlparse(url[0]).netloc for url in data]))
for edr_domain in domains:
# Формируем секцию server
cur.execute("SELECT url FROM edrdata WHERE disabled=0 and url like %s;", ('%://' + edr_domain + '/%',))
edr_urls = cur.fetchall()
# edr_ports = set(['443' if urlparse(i[0]).scheme == 'https' else '80' for i in edr_urls if i[0]])
edr_ports = set([urlparse(i[0].strip()).scheme for i in edr_urls if i[0]])
for edr_port in edr_ports:
cur.execute("SELECT url FROM edrdata WHERE disabled=0 and url like %s;",
edr_port + '://' + edr_domain + '/%')
edr_urls = cur.fetchall()
conf_server = """server {
server_name %(domain)s;
listen %(port)s;
resolver %(dns_serv)s;
""" % {'domain': edr_domain, 'port': '443' if edr_port == 'https' else '80', 'dns_serv': __edr.config('URLS')['dns_serv']}
# Формирует location
conf_location = ""
domain_block = 0
url_string = "/"
for edr_url_temp in edr_urls:
edr_url = urlparse(edr_url_temp[0].strip())
# domain_block = 0 if (edr_url.path and (not edr_url.path == '/')) else 1
if (not edr_url.path) or (edr_url.path == '/'):
domain_block = 1
if (edr_url.scheme+edr_url.netloc).__len__()+3 != edr_url_temp[0].strip().__len__():
url_string = edr_url_temp[0].strip()[(edr_url.scheme+edr_url.netloc).__len__()+3:]
# if edr_url_temp[0].strip().__contains__(" ") and re.search('[А-Я]+', edr_url_temp[0]):
# url_string = quote(url_string).replace('%3D', '=') \
# .replace('%26', '&').replace('%23', '#').replace('%3F', '?')
conf_location += """ location "%s" {
proxy_pass %s;
}
""" % (url_string, __edr.config('URLS')['nginx_stop_url'])
if not domain_block:
conf_location += """ location / {
proxy_pass http://$host;
}
"""
# Закрываем настройки сервера
conf_end = """}
"""
__edr.printt(conf_server + conf_location + conf_end)
nginx_conf_file.write(conf_server + conf_location + conf_end)
# для одиночных доменов, без урлов
cur.execute("SELECT url FROM edrdata WHERE disabled=0 GROUP BY domain;")
data = cur.fetchall()
domains = sorted(set([urlparse(url[0]).netloc for url in data]))
for edr_domain in domains:
# Формируем секцию server
if not edr_domain:
continue
cur.execute("SELECT url FROM edrdata WHERE disabled=0 and url like %s;", ('%://' + edr_domain,))
edr_urls = cur.fetchall()
if not edr_urls:
continue
edr_port = urlparse(edr_urls[0][0].strip()).scheme if edr_urls[0][0] else "http"
conf_server = """server {
server_name %(domain)s;
listen %(port)s;
""" % {'domain': edr_domain, 'port': '443' if edr_port == 'https' else '80'}
# Формирует location
conf_location = """ location / {
proxy_pass %s;
}
""" % (__edr.config('URLS')['nginx_stop_url'])
# Закрываем настройки сервера
conf_end = """}
"""
__edr.printt(conf_server + conf_location + conf_end)
nginx_conf_file.write(conf_server + conf_location + conf_end)
nginx_conf_file.close()
