예제 #1
0
    def load_proxies(self):
        proxy_file = self.proxy_file

        if proxy_file:
            if not os.path.exists(proxy_file):
                raise Exception('proxy_file not found: {0}'.format(proxy_file))

            for line in common.read_lines(proxy_file):
                if 'proxy_auth' in line:
                    self.proxy_auth = common.DataItem(line).rr(
                        'proxy_auth\s*=\s*').trim()
                    continue

                #support tab, commas separator as well
                line = line.replace('\t', ':').replace(',', ':')
                self.proxies.append(line)

        return self
예제 #2
0
파일: http.py 프로젝트: cungnv/scrapex
	def load_proxies(self):     
		proxy_file = self.proxy_file

		if proxy_file:
			if not os.path.exists(proxy_file):
				raise Exception('proxy_file not found: {0}'.format(proxy_file))
			

			for line in common.read_lines(proxy_file):
				if 'proxy_auth' in line:
					self.proxy_auth = common.DataItem(line).rr('proxy_auth\s*=\s*').trim()
					continue

				#support tab, commas separator as well
				line = line.replace('\t',':').replace(',',':')  
				self.proxies.append(self.parse_proxy(line))     


		return self 
예제 #3
0
    def load_oneproxy(self):
        proxy_file = self.proxy_file

        if proxy_file:
            if not os.path.exists(proxy_file):
                raise Exception('proxy_file not found: {0}'.format(proxy_file))

            proxy_auth_found = False
            proxy_selected = False

            lines = common.read_lines(proxy_file)
            while (True):
                line = random.choice(lines)
                if 'proxy_auth' in line:
                    if (proxy_auth_found is False):
                        self.proxy_auth = common.DataItem(line).rr(
                            'proxy_auth\s*=\s*').trim()
                        proxy_auth_found = True
                else:
                    if (proxy_selected is False):
                        #support tab, commas separator as well
                        line = line.replace('\t', ':').replace(',', ':')
                        self.proxies.append(line)
                        proxy_selected = True

                if (proxy_selected):
                    if (proxy_auth_found):
                        break
                    else:
                        for line in lines:
                            if 'proxy_auth' in line:
                                self.proxy_auth = common.DataItem(line).rr(
                                    'proxy_auth\s*=\s*').trim()
                                proxy_auth_found = True
                                break
                        if (proxy_auth_found):
                            break
        return self
예제 #4
0
파일: core.py 프로젝트: igizm0/scrapex
 def read_lines(self, filename):
     return common.read_lines(self.join_path(filename))
예제 #5
0
파일: core.py 프로젝트: cungnv/scrapex
	def read_lines(self, filename):
		return common.read_lines(self.join_path(filename))