def _get_parter_offer_url(self, response): partner_offer_js_list = filter( lambda x: 'getJSON' in x.extract(), response.xpath('//script[@type="text/javascript"]')) if len(partner_offer_js_list) == 1: partner_offer_js = partner_offer_js_list[0].xpath( 'text()').extract()[0] # very ugly fast implementation => regex should be used start_params_pos = partner_offer_js.find('getJSON') start_params_pos = start_params_pos + len('getJSON') self.logger.debug(start_params_pos) end_dict_pos = partner_offer_js.find('}', start_params_pos) assert end_dict_pos > start_params_pos first_comma = partner_offer_js.find(',', start_params_pos) first_param = partner_offer_js[ start_params_pos:first_comma].replace("'", "").strip()[1:] param_dict_string = partner_offer_js[first_comma + 1:end_dict_pos + 1].strip() query_string = jsonurl.query_string(json.loads(param_dict_string)) url = first_param + '?' + query_string return url.strip() elif len(partner_offer_js_list) == 0: self.logger.debug("No js fot partner offers") else: raise ParseError
def _get_parter_offer_url(self, response): partner_offer_js_list = filter(lambda x: 'getJSON' in x.extract(), response.xpath('//script[@type="text/javascript"]')) if len(partner_offer_js_list) == 1: partner_offer_js = partner_offer_js_list[0].xpath('text()').extract()[0] # very ugly fast implementation => regex should be used start_params_pos = partner_offer_js.find('getJSON') start_params_pos = start_params_pos + len('getJSON') self.logger.debug(start_params_pos) end_dict_pos = partner_offer_js.find('}', start_params_pos) assert end_dict_pos > start_params_pos first_comma = partner_offer_js.find(',', start_params_pos) first_param = partner_offer_js[start_params_pos: first_comma].replace("'","").strip()[1:] param_dict_string = partner_offer_js[first_comma+1: end_dict_pos+1].strip() query_string = jsonurl.query_string(json.loads(param_dict_string)) url = first_param +'?' + query_string return url.strip() elif len(partner_offer_js_list) == 0: self.logger.debug("No js fot partner offers") else: raise ParseError
def encodeQS(self): return jsonurl.query_string(self.query)
def encodeQS(self): return jsonurl.query_string(self.query).replace('..', '.')
def setUpClass(cls): d = {"one": 1, "two": 2} cls.query_string = jsonurl.query_string(d)
def setUpClass(cls): d = {"b": "last", "a": [1, 2, 3]} cls.query_string = jsonurl.query_string(d)
def setUpClass(cls): cls.d = {"user.names" : ["richard", "jones"]} cls.query_string = jsonurl.query_string(cls.d) cls.parsed_query = jsonurl.parse_query(cls.query_string)
def setUpClass(cls): cls.d = {"escape_me" : "I'll need escaping"} cls.query_string = jsonurl.query_string(cls.d) cls.parsed_query = jsonurl.parse_query(cls.query_string)
def setUpClass(cls): d = {"one" : [ {"two" : 2, "three" : 3}, 4 ]} cls.query_string = jsonurl.query_string(d)
def setUpClass(cls): cls.d = {"one": 1, "two": [2, 3, 4]} cls.query_string = jsonurl.query_string(cls.d) cls.args = jsonurl.dict_to_args(cls.d) cls.parsed_query = jsonurl.parse_query(cls.query_string)
def setUpClass(cls): cls.d = {"one": {"two": 2, "three": 3}, "four": 4} cls.query_string = jsonurl.query_string(cls.d) cls.args = jsonurl.dict_to_args(cls.d) cls.parsed_query = jsonurl.parse_query(cls.query_string)