def _get_parter_offer_url(self, response):
partner_offer_js_list = filter(
lambda x: 'getJSON' in x.extract(),
response.xpath('//script[@type="text/javascript"]'))
if len(partner_offer_js_list) == 1:
partner_offer_js = partner_offer_js_list[0].xpath(
'text()').extract()[0]
# very ugly fast implementation => regex should be used
start_params_pos = partner_offer_js.find('getJSON')
start_params_pos = start_params_pos + len('getJSON')
self.logger.debug(start_params_pos)
end_dict_pos = partner_offer_js.find('}', start_params_pos)
assert end_dict_pos > start_params_pos
first_comma = partner_offer_js.find(',', start_params_pos)
first_param = partner_offer_js[
start_params_pos:first_comma].replace("'", "").strip()[1:]
param_dict_string = partner_offer_js[first_comma + 1:end_dict_pos +
1].strip()
query_string = jsonurl.query_string(json.loads(param_dict_string))
url = first_param + '?' + query_string
return url.strip()
elif len(partner_offer_js_list) == 0:
self.logger.debug("No js fot partner offers")
else:
raise ParseError
def _get_parter_offer_url(self, response):
partner_offer_js_list = filter(lambda x: 'getJSON' in x.extract(), response.xpath('//script[@type="text/javascript"]'))
if len(partner_offer_js_list) == 1:
partner_offer_js = partner_offer_js_list[0].xpath('text()').extract()[0]
# very ugly fast implementation => regex should be used
start_params_pos = partner_offer_js.find('getJSON')
start_params_pos = start_params_pos + len('getJSON')
self.logger.debug(start_params_pos)
end_dict_pos = partner_offer_js.find('}', start_params_pos)
assert end_dict_pos > start_params_pos
first_comma = partner_offer_js.find(',', start_params_pos)
first_param = partner_offer_js[start_params_pos: first_comma].replace("'","").strip()[1:]
param_dict_string = partner_offer_js[first_comma+1: end_dict_pos+1].strip()
query_string = jsonurl.query_string(json.loads(param_dict_string))
url = first_param +'?' + query_string
return url.strip()
elif len(partner_offer_js_list) == 0:
self.logger.debug("No js fot partner offers")
else:
raise ParseError
def encodeQS(self): return jsonurl.query_string(self.query)
def encodeQS(self):
return jsonurl.query_string(self.query).replace('..', '.')
def encodeQS(self):
return jsonurl.query_string(self.query).replace('..', '.')
def setUpClass(cls):
d = {"one": 1, "two": 2}
cls.query_string = jsonurl.query_string(d)
def setUpClass(cls):
d = {"b": "last", "a": [1, 2, 3]}
cls.query_string = jsonurl.query_string(d)
def setUpClass(cls):
cls.d = {"user.names" : ["richard", "jones"]}
cls.query_string = jsonurl.query_string(cls.d)
cls.parsed_query = jsonurl.parse_query(cls.query_string)
def setUpClass(cls):
cls.d = {"escape_me" : "I'll need escaping"}
cls.query_string = jsonurl.query_string(cls.d)
cls.parsed_query = jsonurl.parse_query(cls.query_string)
def setUpClass(cls):
d = {"one" : [ {"two" : 2, "three" : 3}, 4 ]}
cls.query_string = jsonurl.query_string(d)
def setUpClass(cls):
cls.d = {"one": 1, "two": [2, 3, 4]}
cls.query_string = jsonurl.query_string(cls.d)
cls.args = jsonurl.dict_to_args(cls.d)
cls.parsed_query = jsonurl.parse_query(cls.query_string)
def setUpClass(cls):
cls.d = {"one": {"two": 2, "three": 3}, "four": 4}
cls.query_string = jsonurl.query_string(cls.d)
cls.args = jsonurl.dict_to_args(cls.d)
cls.parsed_query = jsonurl.parse_query(cls.query_string)
