def __setstate__(self, state):
"""
Create pycurl instance after Grag instance was restored
from pickled state.
"""
state['kit'] = Kit()
self.__dict__ = state
def __init__(self):
self.kit = Kit()
class KitTransport(object):
"""
Grab network transport powered with QtWebKit module
"""
def __init__(self):
self.kit = Kit()
#def setup_body_file(self, storage_dir, storage_filename):
#if storage_filename is None:
#handle, path = tempfile.mkstemp(dir=storage_dir)
#else:
#path = os.path.join(storage_dir, storage_filename)
#self.body_file = open(path, 'wb')
#self.body_path = path
def reset(self):
self.request_object = {
'url': None,
'cookies': {},
'method': None,
'data': None,
'user_agent': None,
}
self.response = None
#self.response_head_chunks = []
#self.response_body_chunks = []
#self.response_body_bytes_read = 0
#self.verbose_logging = False
#self.body_file = None
#self.body_path = None
## Maybe move to super-class???
self.request_head = ''
self.request_body = ''
self.request_log = ''
def process_config(self, grab):
self.request_object['url'] = grab.config['url']
self.request_object['method'] = grab.request_method.lower()
if grab.config['cookiefile']:
grab.load_cookies(grab.config['cookiefile'])
if grab.config['cookies']:
if not isinstance(grab.config['cookies'], dict):
raise error.GrabMisuseError('cookies option shuld be a dict')
self.request_object['cookies'] = grab.config['cookies']
if grab.request_method == 'POST':
if grab.config['multipart_post']:
raise NotImplementedError
elif grab.config['post']:
post_data = normalize_post_data(grab.config['post'], grab.config['charset'])
else:
post_data = None
self.request_object['data'] = post_data
if grab.config['user_agent'] is None:
if grab.config['user_agent_file'] is not None:
with open(grab.config['user_agent_file']) as inf:
lines = inf.read().splitlines()
grab.config['user_agent'] = random.choice(lines)
else:
pass
# I think that it does not make sense
# to create random user agents for webkit transport
#grab.config['user_agent'] = random_user_agent()
self.request_object['user_agent'] = grab.config['user_agent']
def request(self):
req = self.request_object
self.kit_response = self.kit.request(
url=req['url'],
cookies=req['cookies'],
method=req['method'],
data=req['data'],
user_agent=req['user_agent'],
)
def prepare_response(self, grab):
return self.kit_response
def extract_cookies(self):
"""
Extract cookies.
"""
return self.kit_response.cookies
def __getstate__(self):
"""
Reset curl attribute which could not be pickled.
"""
state = self.__dict__.copy()
state['kit'] = None
return state
def __setstate__(self, state):
"""
Create pycurl instance after Grag instance was restored
from pickled state.
"""
state['kit'] = Kit()
self.__dict__ = state
