def setup_transport(self, transport_param, reset=False): if self.transport is not None and not reset: raise error.GrabMisuseError( 'Transport is already set up. Use' ' setup_transport(..., reset=True) to explicitly setup' ' new transport') if transport_param is None: transport_param = DEFAULT_TRANSPORT if isinstance(transport_param, six.string_types): if transport_param in TRANSPORT_ALIAS: transport_param = TRANSPORT_ALIAS[transport_param] if '.' not in transport_param: raise error.GrabMisuseError('Unknown transport: %s' % transport_param) else: mod_path, cls_name = transport_param.rsplit('.', 1) try: cls = TRANSPORT_CACHE[(mod_path, cls_name)] except KeyError: mod = __import__(mod_path, globals(), locals(), ['foo']) cls = getattr(mod, cls_name) TRANSPORT_CACHE[(mod_path, cls_name)] = cls self.transport = cls() elif isinstance(transport_param, collections.Callable): self.transport = transport_param() else: raise error.GrabMisuseError('Option `transport` should be string ' 'or class or callable. Got %s' % type(transport_param))
def load_proxylist(self, source, source_type, proxy_type='http', auto_init=True, auto_change=True, **kwargs): # self.proxylist = ProxyList(source, source_type, # proxy_type=proxy_type, **kwargs) if source_type == 'text_file': self.proxylist.set_source('file', location=source, proxy_type=proxy_type, **kwargs) elif source_type == 'url': self.proxylist.set_source('url', url=source, proxy_type=proxy_type, **kwargs) else: raise error.GrabMisuseError('Unknown proxy source type: %s' % source_type) # self.proxylist.setup(auto_change=auto_change, auto_init=auto_init) self.setup(proxy_auto_change=auto_change) if not auto_change and auto_init: self.change_proxy()
def process_cookie_options(self, grab, req): # `cookiefile` option should be processed before `cookies` option # because `load_cookies` updates `cookies` option if grab.config['cookiefile']: # Do not raise exception if cookie file does not exist try: grab.cookies.load_from_file(grab.config['cookiefile']) except IOError as ex: logging.error(ex) request_host = urlsplit(req.url).hostname if request_host: if request_host.startswith('www.'): request_host_no_www = request_host[4:] else: request_host_no_www = request_host # Process `cookies` option that is simple dict i.e. # it provides only `name` and `value` attributes of cookie # No domain, no path, no expires, etc # I pass these no-domain cookies to *each* requested domain # by setting these cookies with corresponding domain attribute # Trying to guess better domain name by removing leading "www." if grab.config['cookies']: if not isinstance(grab.config['cookies'], dict): raise error.GrabMisuseError('cookies option should' ' be a dict') for name, value in grab.config['cookies'].items(): grab.cookies.set(name=name, value=value, domain=request_host_no_www) cookie_hdr = grab.cookies.get_cookie_header(req) if cookie_hdr: req.headers['Cookie'] = cookie_hdr
def setup_document(self, content, **kwargs): """ Setup `response` object without real network requests. Useful for testing and debuging. All ``**kwargs`` will be passed to `Document` constructor. """ self.reset() if isinstance(content, six.text_type): raise error.GrabMisuseError('Method `setup_document` accepts only ' 'byte string in `content` argument.') # Configure Document instance doc = Document(grab=self) doc.body = content doc.status = '' doc.head = b'HTTP/1.1 200 OK\r\n\r\n' doc.parse(charset=kwargs.get('document_charset')) doc.code = 200 doc.total_time = 0 doc.connect_time = 0 doc.name_lookup_time = 0 doc.url = '' for key, value in kwargs.items(): setattr(doc, key, value) self.doc = doc
def setup(self, **kwargs): """ Setting up Grab instance configuration. """ if 'hammer_mode' in kwargs: logging.error( 'Option hammer_mode is deprecated. Grab does not support hammer mode anymore.' ) del kwargs['hammer_mode'] if 'hammer_timeouts' in kwargs: logging.error( 'Option hammer_timeouts is deprecated. Grab does not support hammer mode anymore.' ) del kwargs['hammer_timeouts'] for key in kwargs: if not key in self.config.keys(): raise error.GrabMisuseError('Unknown option: %s' % key) if 'url' in kwargs: if self.config.get('url'): kwargs['url'] = self.make_url_absolute(kwargs['url']) self.config.update(kwargs)
def setup(self, **kwargs): """ Setting up Grab instance configuration. """ json_kwarg = kwargs.pop('json', None) if json_kwarg is not None: if 'headers' in kwargs: headers = kwargs['headers'] = kwargs['headers'].copy() else: headers = kwargs['headers'] = {} headers.update({'Content-Type': 'application/json; charset=utf-8'}) kwargs['post'] = json.dumps(json_kwarg) for key in kwargs: if key not in self.config.keys(): raise error.GrabMisuseError('Unknown option: %s' % key) if 'url' in kwargs: if self.config.get('url'): kwargs['url'] = self.make_url_absolute(kwargs['url']) for header_field in ['headers', 'common_headers']: if header_field in kwargs and isinstance(kwargs[header_field], str): headers = {[y.strip() for y in x.strip('> ').split(':', 1)] for x in kwargs[header_field].split('\n') if x.strip(' >')} kwargs[header_field] = headers self.config.update(kwargs)
def process_cookie_options(self, grab, request_url): host = urlsplit(request_url).netloc.split(':')[0] host_nowww = host if host_nowww.startswith('www.'): host_nowww = host_nowww[4:] # `cookiefile` option should be processed before `cookies` option # because `load_cookies` updates `cookies` option if grab.config['cookiefile']: grab.cookies.load_from_file(grab.config['cookiefile']) if grab.config['cookies']: if not isinstance(grab.config['cookies'], dict): raise error.GrabMisuseError('cookies option should be a dict') for name, value in grab.config['cookies'].items(): if '.' in host_nowww: domain = host_nowww else: domain = '' grab.cookies.set( #name=normalize_unicode(name, grab.config['charset']), #value=normalize_unicode(value, grab.config['charset']), name=name, value=value, domain=domain) # Erase known cookies stored in pycurl handler self.curl.setopt(pycurl.COOKIELIST, 'ALL') # Enable pycurl cookie processing mode self.curl.setopt(pycurl.COOKIELIST, '') # TODO: At this point we should use cookielib magic # to pick up cookies for the current requests for cookie in grab.cookies.cookiejar: cookie_domain = cookie.domain http_only = cookie_domain.startswith('#httponly_') if http_only: cookie_domain = cookie_domain.replace('#httponly_', '') if not cookie_domain or host_nowww in cookie_domain: encoded = encode_cookies({cookie.name: cookie.value}, join=True, charset=grab.config['charset']) cookie_string = b'Set-Cookie: ' + encoded if len(cookie.path) != 0: cookie_string += b'; path=' + cookie.path.encode('ascii') if '.' in host_nowww: cookie_string += b'; domain=' + cookie_domain.encode( 'ascii') if http_only: cookie_string += b'; HttpOnly' self.curl.setopt(pycurl.COOKIELIST, cookie_string)
def setup(self, **kwargs): """ Setting up Grab instance configuration. """ for key in kwargs: if key not in self.config.keys(): raise error.GrabMisuseError('Unknown option: %s' % key) if 'url' in kwargs: if self.config.get('url'): kwargs['url'] = self.make_url_absolute(kwargs['url']) self.config.update(kwargs)
def setup_transport(self, transport_param): self.transport_param = transport_param if isinstance(transport_param, six.string_types): if transport_param in TRANSPORT_ALIAS: transport_param = TRANSPORT_ALIAS[transport_param] if not '.' in transport_param: raise error.GrabMisuseError('Unknown transport: %s' % transport_param) else: mod_path, cls_name = transport_param.rsplit('.', 1) try: cls = TRANSPORT_CACHE[(mod_path, cls_name)] except KeyError: mod = __import__(mod_path, globals(), locals(), ['foo']) cls = getattr(mod, cls_name) TRANSPORT_CACHE[(mod_path, cls_name)] = cls self.transport = cls() elif isinstance(transport_param, collections.Callable): self.transport = transport_param() else: raise error.GrabMisuseError('Option `transport` should be string ' 'or callable. Got %s' % type(transport_param))
def process_cookie_options(self, grab, request_url): request_host = urlsplit(request_url).netloc.split(':')[0] if request_host.startswith('www.'): request_host_no_www = request_host[4:] else: request_host_no_www = request_host # `cookiefile` option should be processed before `cookies` option # because `load_cookies` updates `cookies` option if grab.config['cookiefile']: # Do not raise exception if cookie file does not exist try: grab.cookies.load_from_file(grab.config['cookiefile']) except IOError as ex: logging.error(ex) # Process `cookies` option that is simple dict i.e. # it provides only `name` and `value` attributes of cookie # No domain, no path, no expires, etc # I pass these no-domain cookies to *each* requested domain # by setting these cookies with corresponding domain attribute # Trying to guess better domain name by removing leading "www." if grab.config['cookies']: if not isinstance(grab.config['cookies'], dict): raise error.GrabMisuseError('cookies option should be a dict') for name, value in grab.config['cookies'].items(): grab.cookies.set( name=name, value=value, domain=request_host_no_www ) # Erase known cookies stored in pycurl handler self.curl.setopt(pycurl.COOKIELIST, 'ALL') # Enable pycurl cookie processing mode self.curl.setopt(pycurl.COOKIELIST, '') # Put all cookies from `grab.cookies.cookiejar` to # the pycurl instance. # We put *all* cookies, for all host names # Pycurl cookie engine is smart enough to send # only cookies belong to the current request's host name for cookie in grab.cookies.cookiejar: self.curl.setopt(pycurl.COOKIELIST, self.get_netscape_cookie_spec(cookie, request_host))
def setup_transport(self, transport_param): self.transport_param = transport_param if isinstance(transport_param, basestring): mod_path, cls_name = transport_param.rsplit('.', 1) try: cls = TRANSPORT_CACHE[(mod_path, cls_name)] except KeyError: mod = __import__(mod_path, globals(), locals(), ['foo']) cls = getattr(mod, cls_name) TRANSPORT_CACHE[(mod_path, cls_name)] = cls self.transport = cls() elif isinstance(transport_param, collections.Callable): self.transport = transport_param() else: raise error.GrabMisuseError( 'Option `transport` should be string or callable. ' 'Got %s' % type(transport_param))
def load_proxylist(self, source, source_type, proxy_type='http', auto_init=True, auto_change=True): # self.proxylist = ProxyList(source, source_type, # proxy_type=proxy_type, **kwargs) if source_type == 'text_file': # pylint: disable=no-member self.proxylist.load_file(source, proxy_type=proxy_type) # pylint: enable=no-member elif source_type == 'url': # pylint: disable=no-member self.proxylist.load_url(source, proxy_type=proxy_type) # pylint: enable=no-member else: raise error.GrabMisuseError('Unknown proxy source type: %s' % source_type) # self.proxylist.setup(auto_change=auto_change, auto_init=auto_init) self.setup(proxy_auto_change=auto_change) # pylint: disable=no-member if not auto_change and auto_init: self.change_proxy() # pylint: disable=no-member
def process_config(self, grab): """ Setup curl instance with values from ``self.config``. """ # Copy some config for future usage self.config_nobody = grab.config['nobody'] self.config_body_maxsize = grab.config['body_maxsize'] try: request_url = normalize_url(grab.config['url']) except Exception as ex: raise error.GrabInvalidUrl(u'%s: %s' % (six.text_type(ex), grab.config['url'])) # py3 hack if not six.PY3: request_url = make_str(request_url) self.curl.setopt(pycurl.URL, request_url) # 30* redirects are handled by Grab self.curl.setopt(pycurl.FOLLOWLOCATION, 0) self.curl.setopt(pycurl.MAXREDIRS, grab.config['redirect_limit']) self.curl.setopt(pycurl.CONNECTTIMEOUT, grab.config['connect_timeout']) self.curl.setopt(pycurl.TIMEOUT, grab.config['timeout']) #self.curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4) # self.curl.setopt(pycurl.DNS_CACHE_TIMEOUT, 0) if not grab.config['connection_reuse']: self.curl.setopt(pycurl.FRESH_CONNECT, 1) self.curl.setopt(pycurl.FORBID_REUSE, 1) self.curl.setopt(pycurl.NOSIGNAL, 1) self.curl.setopt(pycurl.HEADERFUNCTION, self.header_processor) if grab.config['body_inmemory']: self.curl.setopt(pycurl.WRITEFUNCTION, self.body_processor) else: if not grab.config['body_storage_dir']: raise error.GrabMisuseError( 'Option body_storage_dir is not defined') self.setup_body_file( grab.config['body_storage_dir'], grab.config['body_storage_filename'], create_dir=grab.config['body_storage_create_dir']) self.curl.setopt(pycurl.WRITEFUNCTION, self.body_processor) if grab.config['verbose_logging']: self.verbose_logging = True # User-Agent if grab.config['user_agent'] is None: if grab.config['user_agent_file'] is not None: with open(grab.config['user_agent_file']) as inf: lines = inf.read().splitlines() grab.config['user_agent'] = random.choice(lines) else: grab.config['user_agent'] = generate_user_agent() # If value is None then set empty string # None is not acceptable because in such case # pycurl will set its default user agent "PycURL/x.xx.x" if not grab.config['user_agent']: grab.config['user_agent'] = '' self.curl.setopt(pycurl.USERAGENT, grab.config['user_agent']) if grab.config['debug']: self.curl.setopt(pycurl.VERBOSE, 1) self.curl.setopt(pycurl.DEBUGFUNCTION, self.debug_processor) # Ignore SSL errors self.curl.setopt(pycurl.SSL_VERIFYPEER, 0) self.curl.setopt(pycurl.SSL_VERIFYHOST, 0) # Disabled to avoid SSL3_READ_BYTES:sslv3 alert handshake failure error # self.curl.setopt(pycurl.SSLVERSION, pycurl.SSLVERSION_SSLv3) if grab.request_method in ('POST', 'PUT'): if (grab.config['post'] is None and grab.config['multipart_post'] is None): raise GrabMisuseError('Neither `post` or `multipart_post`' ' options was specified for the %s' ' request' % grab.request_method) if grab.request_method == 'POST': self.curl.setopt(pycurl.POST, 1) if grab.config['multipart_post']: if isinstance(grab.config['multipart_post'], six.string_types): raise error.GrabMisuseError( 'multipart_post option could not be a string') post_items = normalize_http_values( grab.config['multipart_post'], charset=grab.config['charset'], ignore_classes=(UploadFile, UploadContent), ) # py3 hack #if six.PY3: # post_items = decode_pairs(post_items, # grab.config['charset']) self.curl.setopt(pycurl.HTTPPOST, process_upload_items(post_items)) elif grab.config['post']: post_data = normalize_post_data(grab.config['post'], grab.config['charset']) # py3 hack # if six.PY3: # post_data = smart_unicode(post_data, # grab.config['charset']) self.curl.setopt(pycurl.POSTFIELDS, post_data) else: self.curl.setopt(pycurl.POSTFIELDS, '') elif grab.request_method == 'PUT': data = grab.config['post'] if isinstance(data, six.text_type): # py3 hack # if six.PY3: # data = data.encode('utf-8') # else: raise error.GrabMisuseError( 'Value of post option could be only ' 'byte string if PUT method is used') self.curl.setopt(pycurl.UPLOAD, 1) self.curl.setopt(pycurl.CUSTOMREQUEST, 'PUT') self.curl.setopt(pycurl.READFUNCTION, StringIO(data).read) self.curl.setopt(pycurl.INFILESIZE, len(data)) elif grab.request_method == 'PATCH': data = grab.config['post'] if isinstance(data, six.text_type): raise error.GrabMisuseError( 'Value of post option could be only byte ' 'string if PATCH method is used') self.curl.setopt(pycurl.UPLOAD, 1) self.curl.setopt(pycurl.CUSTOMREQUEST, 'PATCH') self.curl.setopt(pycurl.READFUNCTION, StringIO(data).read) self.curl.setopt(pycurl.INFILESIZE, len(data)) elif grab.request_method == 'DELETE': self.curl.setopt(pycurl.CUSTOMREQUEST, 'DELETE') elif grab.request_method == 'HEAD': self.curl.setopt(pycurl.NOBODY, 1) elif grab.request_method == 'UPLOAD': self.curl.setopt(pycurl.UPLOAD, 1) elif grab.request_method == 'GET': self.curl.setopt(pycurl.HTTPGET, 1) elif grab.request_method == 'OPTIONS': data = grab.config['post'] if data is not None: if isinstance(data, six.text_type): raise error.GrabMisuseError( 'Value of post option could be only byte ' 'string if PATCH method is used') self.curl.setopt(pycurl.UPLOAD, 1) self.curl.setopt(pycurl.READFUNCTION, StringIO(data).read) self.curl.setopt(pycurl.INFILESIZE, len(data)) self.curl.setopt(pycurl.CUSTOMREQUEST, 'OPTIONS') else: raise error.GrabMisuseError('Invalid method: %s' % grab.request_method) headers = grab.config['common_headers'] if grab.config['headers']: headers.update(grab.config['headers']) # This is required to avoid some problems headers.update({'Expect': ''}) header_tuples = [str('%s: %s' % x) for x in headers.items()] self.curl.setopt(pycurl.HTTPHEADER, header_tuples) self.process_cookie_options(grab, request_url) if grab.config['referer']: self.curl.setopt(pycurl.REFERER, str(grab.config['referer'])) if grab.config['proxy']: self.curl.setopt(pycurl.PROXY, str(grab.config['proxy'])) else: self.curl.setopt(pycurl.PROXY, '') if grab.config['proxy_userpwd']: self.curl.setopt(pycurl.PROXYUSERPWD, str(grab.config['proxy_userpwd'])) if grab.config['proxy_type']: key = 'PROXYTYPE_%s' % grab.config['proxy_type'].upper() self.curl.setopt(pycurl.PROXYTYPE, getattr(pycurl, key)) if grab.config['encoding']: if ('gzip' in grab.config['encoding'] and 'zlib' not in pycurl.version): raise error.GrabMisuseError( 'You can not use gzip encoding because ' 'pycurl was built without zlib support') self.curl.setopt(pycurl.ENCODING, grab.config['encoding']) if grab.config['userpwd']: self.curl.setopt(pycurl.USERPWD, str(grab.config['userpwd'])) if grab.config.get('interface') is not None: self.curl.setopt(pycurl.INTERFACE, grab.config['interface']) if grab.config.get('reject_file_size') is not None: self.curl.setopt(pycurl.MAXFILESIZE, grab.config['reject_file_size'])
def process_config(self, grab): """ Setup curl instance with values from ``self.config``. """ # Copy some config for future usage self.config_nobody = grab.config['nobody'] self.config_body_maxsize = grab.config['body_maxsize'] try: request_url = normalize_url(grab.config['url']) except Exception as ex: raise error.GrabInvalidUrl(u'%s: %s' % (unicode(ex), grab.config['url'])) # py3 hack if not PY3K: request_url = smart_str(request_url) self.curl.setopt(pycurl.URL, request_url) self.curl.setopt(pycurl.FOLLOWLOCATION, 1 if grab.config['follow_location'] else 0) self.curl.setopt(pycurl.MAXREDIRS, grab.config['redirect_limit']) self.curl.setopt(pycurl.CONNECTTIMEOUT, grab.config['connect_timeout']) self.curl.setopt(pycurl.TIMEOUT, grab.config['timeout']) self.curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4) #self.curl.setopt(pycurl.DNS_CACHE_TIMEOUT, 0) if not grab.config['connection_reuse']: self.curl.setopt(pycurl.FRESH_CONNECT, 1) self.curl.setopt(pycurl.FORBID_REUSE, 1) self.curl.setopt(pycurl.NOSIGNAL, 1) self.curl.setopt(pycurl.HEADERFUNCTION, self.head_processor) if grab.config['body_inmemory']: self.curl.setopt(pycurl.WRITEFUNCTION, self.body_processor) else: if not grab.config['body_storage_dir']: raise error.GrabMisuseError( 'Option body_storage_dir is not defined') self.setup_body_file(grab.config['body_storage_dir'], grab.config['body_storage_filename']) self.curl.setopt(pycurl.WRITEFUNCTION, self.body_processor) if grab.config['verbose_logging']: self.verbose_logging = True # User-Agent if grab.config['user_agent'] is None: if grab.config['user_agent_file'] is not None: with open(grab.config['user_agent_file']) as inf: lines = inf.read().splitlines() grab.config['user_agent'] = random.choice(lines) else: grab.config['user_agent'] = random_user_agent() # If value is None then set empty string # None is not acceptable because in such case # pycurl will set its default user agent "PycURL/x.xx.x" if not grab.config['user_agent']: grab.config['user_agent'] = '' self.curl.setopt(pycurl.USERAGENT, grab.config['user_agent']) if grab.config['debug']: self.curl.setopt(pycurl.VERBOSE, 1) self.curl.setopt(pycurl.DEBUGFUNCTION, self.debug_processor) # Ignore SSL errors self.curl.setopt(pycurl.SSL_VERIFYPEER, 0) self.curl.setopt(pycurl.SSL_VERIFYHOST, 0) self.curl.setopt(pycurl.SSLVERSION, pycurl.SSLVERSION_SSLv3) if grab.request_method == 'POST': self.curl.setopt(pycurl.POST, 1) if grab.config['multipart_post']: if isinstance(grab.config['multipart_post'], basestring): raise error.GrabMisuseError( 'multipart_post option could not be a string') post_items = normalize_http_values( grab.config['multipart_post'], charset=grab.config['charset']) # py3 hack if PY3K: post_items = decode_pairs(post_items, grab.config['charset']) #import pdb; pdb.set_trace() self.curl.setopt(pycurl.HTTPPOST, post_items) elif grab.config['post']: post_data = normalize_post_data(grab.config['post'], grab.config['charset']) # py3 hack #if PY3K: # post_data = smart_unicode(post_data, grab.config['charset']) self.curl.setopt(pycurl.COPYPOSTFIELDS, post_data) else: self.curl.setopt(pycurl.POSTFIELDS, '') elif grab.request_method == 'PUT': data = grab.config['post'] if isinstance(data, unicode) or (not PY3K and not isinstance(data, basestring)): # py3 hack #if PY3K: # data = data.encode('utf-8') #else: raise error.GrabMisuseError('Value of post option could be only '\ 'byte string if PUT method is used') self.curl.setopt(pycurl.UPLOAD, 1) self.curl.setopt(pycurl.READFUNCTION, StringIO(data).read) self.curl.setopt(pycurl.INFILESIZE, len(data)) elif grab.request_method == 'PATCH': data = grab.config['post'] if isinstance(data, unicode) or not isinstance(data, basestring): # py3 hack if PY3K: data = data.encode('utf-8') else: raise error.GrabMisuseError('Value of post option could be only byte '\ 'string if PATCH method is used') self.curl.setopt(pycurl.UPLOAD, 1) self.curl.setopt(pycurl.CUSTOMREQUEST, 'PATCH') self.curl.setopt(pycurl.READFUNCTION, StringIO(data).read) self.curl.setopt(pycurl.INFILESIZE, len(data)) elif grab.request_method == 'DELETE': self.curl.setopt(pycurl.CUSTOMREQUEST, 'delete') elif grab.request_method == 'HEAD': self.curl.setopt(pycurl.NOBODY, 1) elif grab.request_method == 'UPLOAD': self.curl.setopt(pycurl.UPLOAD, 1) elif grab.request_method == 'GET': self.curl.setopt(pycurl.HTTPGET, 1) else: raise error.GrabMisuseError('Invalid method: %s' % grab.request_method) headers = grab.config['common_headers'] if grab.config['headers']: headers.update(grab.config['headers']) header_tuples = [str('%s: %s' % x) for x\ in headers.items()] self.curl.setopt(pycurl.HTTPHEADER, header_tuples) self.process_cookie_options(grab, request_url) if grab.config['referer']: self.curl.setopt(pycurl.REFERER, str(grab.config['referer'])) if grab.config['proxy']: self.curl.setopt(pycurl.PROXY, str(grab.config['proxy'])) else: self.curl.setopt(pycurl.PROXY, '') if grab.config['proxy_userpwd']: self.curl.setopt(pycurl.PROXYUSERPWD, str(grab.config['proxy_userpwd'])) if grab.config['proxy_type']: ptype = getattr(pycurl, 'PROXYTYPE_%s' % grab.config['proxy_type'].upper()) self.curl.setopt(pycurl.PROXYTYPE, ptype) if grab.config['encoding']: if 'gzip' in grab.config[ 'encoding'] and not 'zlib' in pycurl.version: raise error.GrabMisuseError('You can not use gzip encoding because '\ 'pycurl was built without zlib support') self.curl.setopt(pycurl.ENCODING, grab.config['encoding']) if grab.config['userpwd']: self.curl.setopt(pycurl.USERPWD, str(grab.config['userpwd'])) if grab.config.get('interface') is not None: self.curl.setopt(pycurl.INTERFACE, grab.config['interface']) if grab.config.get('reject_file_size') is not None: self.curl.setopt(pycurl.MAXFILESIZE, grab.config['reject_file_size'])