示例#1
0
    def check_for_valid(self, lst):
        '''
		Проверка на валидность
		'''
        lst2 = []
        output = []
        ##############
        for i in lst:
            try:
                if re.match(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d+",
                            i.normal):  # если айпи подходит под шаблон
                    lst2.append(i)
                else:
                    print(self.NAME + f"Found invalid proxy: {i}")
            except Exception as e:
                print(self.NAME + f"Critical error {e}, send BUGREPORT file.")
                tools.log(e, "removeshit", line=25)
                raise e
        #################################
        for i in lst2:
            for i2 in i.octets:
                if (int(i2) > 255
                        or int(i2) < 0) or (len(i2) >= 2 and
                                            (i2[0] == "0" or i2[0:2] == "00")):
                    print(self.NAME + f"Found invalid proxy octet: {i2}")
                    break
            else:
                output.append(i)
        ###############
        return output
示例#2
0
 def addressOutNetwork(self, proxylist):
     output = []
     for i in range(0, len(proxylist)):
         for subnet in self.subnets:
             try:
                 if ipaddress.ip_address(
                         proxylist[i].host) in ipaddress.ip_network(subnet):
                     break
                 else:
                     pass
             except ValueError:
                 continue
             except Exception as e:
                 print(self.NAME + f"Error: {e}")
                 tools.log(e, "sunets", name="ошибка в проверке адреса")
                 raise e
                 break
         else:
             print(
                 self.NAME + colorama.Fore.GREEN +
                 f"Removed host in non-whitelist subnet: {proxylist[i].host}"
             )
             continue
         ###########################
         output.append(proxylist[i])
     return output
示例#3
0
 def addressInNetwork(self, proxylist):
     output = []
     for i in range(0, len(proxylist)):
         for subnet in self.subnets:
             try:
                 if ipaddress.ip_address(
                         proxylist[i].host) in ipaddress.ip_network(subnet):
                     print(
                         self.NAME + colorama.Fore.YELLOW +
                         f"Removed host in blacklist subnet: {proxylist[i].host}"
                     )
                     break
                 else:
                     pass
             except ValueError:
                 print(self.NAME +
                       f"Removed invalid host: {proxylist[i].host}")
                 break
             except Exception as e:
                 print(self.NAME + f"Error: {e}")
                 tools.log(e, "sunets", name="ошибка в проверке адреса")
                 raise e
                 break
         else:
             output.append(proxylist[i])
     return output
示例#4
0
	def zipExtractor(self):
		'''
		Extracting zip files
		'''
		os.chdir("downloads")
		files = os.listdir()
		########################
		for i in files:
			###################
			if not ".zip" in i:
				os.chmod(i, 0o777)
				os.remove(i)
				continue
			###################
			try:
				xfile = zipfile.ZipFile(i)
				xfile.extractall()
				xfile.close()
			except Exception as e:
				os.remove(i)
				print(self.NAME + colorama.Fore.RED + f"Error while extracting archive {i}")
				#raise e
				continue
			else:
				print(self.NAME + colorama.Fore.GREEN + f"Successfully extracted archive {i}")
			##################
			# list of extracted files
			extracted = os.listdir()
			##################
			for l in extracted:
				# parsing proxies from txt files
				if ".txt" in l:
					self.loadProxies_(l)
			##################
			# removing shit
			for j in extracted:
				os.chmod(j, 0o777)
				if ".zip" in j:
					continue
				else:
					try:
						os.remove(j)
					except IsADirectoryError:
						rmtree(j)
					except Exception as e:
						print(self.NAME + colorama.Fore.RED + f"Error {e} while deleting file {j}")
						tools.log(e, "zipExtractor")
			#################
			os.remove(i)
			#################
		os.chdir("..")
		try:
			os.rmdir("downloads")
		except OSError:
			rmtree("downloads")
		except Exception as e:
			tools.log(e, "remove downloads")
示例#5
0
 def start(self):
     print(self.NAME + colorama.Fore.GREEN + "Deleting invalid proxies...")
     try:
         self.proxies = self.check_for_valid(self.proxies)
     except KeyboardInterrupt:
         print(self.NAME + colorama.Fore.RED +
               "Deleting invalid proxies cancelled!")
     except Exception as e:
         print(self.NAME + f"Critical error {e}, send BUGREPORT file.")
         tools.log(e, "removeshit", name="сломалось в старте")
         raise e
     #################
     if self.same:
         try:
             self.proxies = self.checksame(self.proxies)
         except KeyboardInterrupt:
             print(self.NAME + colorama.Fore.RED +
                   "Deleting proxies with same hosts cancelled!")
         except Exception as e:
             raise e
     print(self.NAME + colorama.Fore.GREEN + "Clearing proxies finished!")
     return self.proxies
示例#6
0
	def main(self):
		'''
		Start function for start parser object
		'''
		print(self.NAME + colorama.Fore.GREEN + "Started parse...")
		##############################################
		self.inputLinks = copy.deepcopy(self.MAINURLS)
		self.lock = asyncio.Lock()
		loop = asyncio.get_event_loop()
		tasks = []
		ignore_aiohttp_ssl_eror(loop)
		#############################
		for i in range(0, 3):
			##############################
			for i in range(0, self.TASKS):
				tasks.append(loop.create_task(self.getWebPage()))
			####
			try:
				loop_response = loop.run_until_complete(asyncio.gather(*tasks, return_exceptions=True))
			except Exception as e:
				if isinstance(e, (KeyboardInterrupt, RuntimeError, AttributeError)):
					print("\n" + self.NAME + colorama.Fore.YELLOW + "Parsing stopped! Cancelling all tasks...")
					for i in tasks:
						i.cancel()
					break
				else:
					raise e
			finally:
				self.inputLinks.clear()
				self.inputLinks = copy.deepcopy(self.links)
				self.links.clear()
		#########################################
			for i in self.links:
				if "zip" in i:
					self.zipLinks.add(i)
				else:
					self.inputLinks.add(i)
		#########################################
		print(self.NAME + colorama.Fore.GREEN + "Analyze founded URLs...")
		self.inputLinks = copy.deepcopy(self.zipLinks)
		##############################################
		for i in range(0, self.TASKS):
				tasks.append(loop.create_task(self.getWebPage()))
		#########################################################
		try:
			loop_response = loop.run_until_complete(asyncio.gather(*tasks, return_exceptions=True))
		except Exception as e:
			if isinstance(e, (KeyboardInterrupt, RuntimeError, AttributeError)):
				print("\n" + self.NAME + colorama.Fore.YELLOW + "Parsing stopped! Cancelling all tasks...")
				for i in tasks:
					i.cancel()
			else:
				raise e
		else:
			tools.loopResponse(loop_response, "parser archives")
		#############################################################
		# downloading archives
		self.inputLinks = copy.deepcopy(self.linksForDownload)
		print(self.NAME + colorama.Fore.GREEN + "Downloading .zip archives...")
		#############################################################
		for i in range(0, self.TASKS):
				tasks.append(loop.create_task(self.DownloadArchive()))
		#############################################################
		try:
			loop_response = loop.run_until_complete(asyncio.gather(*tasks, return_exceptions=True))
		except Exception as e:
			if isinstance(e, (KeyboardInterrupt, RuntimeError, AttributeError)):
				print("\n" + self.NAME + colorama.Fore.YELLOW + "Parsing stopped! Cancelling all tasks...")
				for i in tasks:
					i.cancel()
			else:
				raise e
		else:
			tools.loopResponse(loop_response, "download archives")
		###################################################
		try:
			self.zipExtractor()
		except Exception as e:
			if "downloads" in os.getcwd():
				os.chdir("..")
			if os.access("downloads", mode=0):
				rmtree("downloads")
			tools.log(e, "open archives")
		#################################################################
		print(self.NAME + colorama.Fore.GREEN + "Finished parsing proxies.")
		print(self.NAME + f"Got {str(len(self.proxies))} proxies.")
		self.proxies = [*self.proxies]
		#################################################################
		return self.proxies