def check_for_valid(self, lst):
'''
Проверка на валидность
'''
lst2 = []
output = []
##############
for i in lst:
try:
if re.match(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d+",
i.normal): # если айпи подходит под шаблон
lst2.append(i)
else:
print(self.NAME + f"Found invalid proxy: {i}")
except Exception as e:
print(self.NAME + f"Critical error {e}, send BUGREPORT file.")
tools.log(e, "removeshit", line=25)
raise e
#################################
for i in lst2:
for i2 in i.octets:
if (int(i2) > 255
or int(i2) < 0) or (len(i2) >= 2 and
(i2[0] == "0" or i2[0:2] == "00")):
print(self.NAME + f"Found invalid proxy octet: {i2}")
break
else:
output.append(i)
###############
return output
def addressOutNetwork(self, proxylist):
output = []
for i in range(0, len(proxylist)):
for subnet in self.subnets:
try:
if ipaddress.ip_address(
proxylist[i].host) in ipaddress.ip_network(subnet):
break
else:
pass
except ValueError:
continue
except Exception as e:
print(self.NAME + f"Error: {e}")
tools.log(e, "sunets", name="ошибка в проверке адреса")
raise e
break
else:
print(
self.NAME + colorama.Fore.GREEN +
f"Removed host in non-whitelist subnet: {proxylist[i].host}"
)
continue
###########################
output.append(proxylist[i])
return output
def addressInNetwork(self, proxylist):
output = []
for i in range(0, len(proxylist)):
for subnet in self.subnets:
try:
if ipaddress.ip_address(
proxylist[i].host) in ipaddress.ip_network(subnet):
print(
self.NAME + colorama.Fore.YELLOW +
f"Removed host in blacklist subnet: {proxylist[i].host}"
)
break
else:
pass
except ValueError:
print(self.NAME +
f"Removed invalid host: {proxylist[i].host}")
break
except Exception as e:
print(self.NAME + f"Error: {e}")
tools.log(e, "sunets", name="ошибка в проверке адреса")
raise e
break
else:
output.append(proxylist[i])
return output
def zipExtractor(self):
'''
Extracting zip files
'''
os.chdir("downloads")
files = os.listdir()
########################
for i in files:
###################
if not ".zip" in i:
os.chmod(i, 0o777)
os.remove(i)
continue
###################
try:
xfile = zipfile.ZipFile(i)
xfile.extractall()
xfile.close()
except Exception as e:
os.remove(i)
print(self.NAME + colorama.Fore.RED + f"Error while extracting archive {i}")
#raise e
continue
else:
print(self.NAME + colorama.Fore.GREEN + f"Successfully extracted archive {i}")
##################
# list of extracted files
extracted = os.listdir()
##################
for l in extracted:
# parsing proxies from txt files
if ".txt" in l:
self.loadProxies_(l)
##################
# removing shit
for j in extracted:
os.chmod(j, 0o777)
if ".zip" in j:
continue
else:
try:
os.remove(j)
except IsADirectoryError:
rmtree(j)
except Exception as e:
print(self.NAME + colorama.Fore.RED + f"Error {e} while deleting file {j}")
tools.log(e, "zipExtractor")
#################
os.remove(i)
#################
os.chdir("..")
try:
os.rmdir("downloads")
except OSError:
rmtree("downloads")
except Exception as e:
tools.log(e, "remove downloads")
def start(self):
print(self.NAME + colorama.Fore.GREEN + "Deleting invalid proxies...")
try:
self.proxies = self.check_for_valid(self.proxies)
except KeyboardInterrupt:
print(self.NAME + colorama.Fore.RED +
"Deleting invalid proxies cancelled!")
except Exception as e:
print(self.NAME + f"Critical error {e}, send BUGREPORT file.")
tools.log(e, "removeshit", name="сломалось в старте")
raise e
#################
if self.same:
try:
self.proxies = self.checksame(self.proxies)
except KeyboardInterrupt:
print(self.NAME + colorama.Fore.RED +
"Deleting proxies with same hosts cancelled!")
except Exception as e:
raise e
print(self.NAME + colorama.Fore.GREEN + "Clearing proxies finished!")
return self.proxies
def main(self):
'''
Start function for start parser object
'''
print(self.NAME + colorama.Fore.GREEN + "Started parse...")
##############################################
self.inputLinks = copy.deepcopy(self.MAINURLS)
self.lock = asyncio.Lock()
loop = asyncio.get_event_loop()
tasks = []
ignore_aiohttp_ssl_eror(loop)
#############################
for i in range(0, 3):
##############################
for i in range(0, self.TASKS):
tasks.append(loop.create_task(self.getWebPage()))
####
try:
loop_response = loop.run_until_complete(asyncio.gather(*tasks, return_exceptions=True))
except Exception as e:
if isinstance(e, (KeyboardInterrupt, RuntimeError, AttributeError)):
print("\n" + self.NAME + colorama.Fore.YELLOW + "Parsing stopped! Cancelling all tasks...")
for i in tasks:
i.cancel()
break
else:
raise e
finally:
self.inputLinks.clear()
self.inputLinks = copy.deepcopy(self.links)
self.links.clear()
#########################################
for i in self.links:
if "zip" in i:
self.zipLinks.add(i)
else:
self.inputLinks.add(i)
#########################################
print(self.NAME + colorama.Fore.GREEN + "Analyze founded URLs...")
self.inputLinks = copy.deepcopy(self.zipLinks)
##############################################
for i in range(0, self.TASKS):
tasks.append(loop.create_task(self.getWebPage()))
#########################################################
try:
loop_response = loop.run_until_complete(asyncio.gather(*tasks, return_exceptions=True))
except Exception as e:
if isinstance(e, (KeyboardInterrupt, RuntimeError, AttributeError)):
print("\n" + self.NAME + colorama.Fore.YELLOW + "Parsing stopped! Cancelling all tasks...")
for i in tasks:
i.cancel()
else:
raise e
else:
tools.loopResponse(loop_response, "parser archives")
#############################################################
# downloading archives
self.inputLinks = copy.deepcopy(self.linksForDownload)
print(self.NAME + colorama.Fore.GREEN + "Downloading .zip archives...")
#############################################################
for i in range(0, self.TASKS):
tasks.append(loop.create_task(self.DownloadArchive()))
#############################################################
try:
loop_response = loop.run_until_complete(asyncio.gather(*tasks, return_exceptions=True))
except Exception as e:
if isinstance(e, (KeyboardInterrupt, RuntimeError, AttributeError)):
print("\n" + self.NAME + colorama.Fore.YELLOW + "Parsing stopped! Cancelling all tasks...")
for i in tasks:
i.cancel()
else:
raise e
else:
tools.loopResponse(loop_response, "download archives")
###################################################
try:
self.zipExtractor()
except Exception as e:
if "downloads" in os.getcwd():
os.chdir("..")
if os.access("downloads", mode=0):
rmtree("downloads")
tools.log(e, "open archives")
#################################################################
print(self.NAME + colorama.Fore.GREEN + "Finished parsing proxies.")
print(self.NAME + f"Got {str(len(self.proxies))} proxies.")
self.proxies = [*self.proxies]
#################################################################
return self.proxies