def spider_opened(self, spider): if hasattr(spider, 'proxyserver') and spider.proxyserver: if Utils.is_valid_url(spider.proxyserver): self.settings['CUSTOM_PROXY_URL'] = spider.proxyserver else: raise CloseSpider("Proxyserver url({}) not valid.".format( spider.proxyserver))
def run(self, args, opts): if len(args) != 1: raise UsageError() url = args[0] if not Utils.is_valid_url(url): print('please use valid url-{}'.format(url)) return request = urllib.request.Request(url) response = urllib.request.urlopen(request, cafile=certifi.where()) body = Utils.parse_form(response.read().decode(response.headers.get_content_charset())) if not body or len(body) < 0: print('no form is present in {}'.format(url)) print(json.dumps(body, indent=4))