def get(self, *params, response):
data = JSONBase.get(self, *params, json = response.meta['json'])
try:
return HTMLParser().unescape(data)
except:
try:
result = []
for p in data:
result.append(HTMLParser().unescape(p))
return result
except:
return data
def getLRMI(self, *params, response):
try:
lrmi = list(
map(
lambda x: json.loads(
x.replace('\r', '').replace('\n', ' ')),
response.xpath(self.lrmi_path).getall()))
except:
logging.warning('failed parsing lrmi at ' + response.url +
', please check source')
return None
for l in lrmi:
value = JSONBase.get(self, *params, json=l)
if value != None:
return HTMLParser().unescape(value)
return None
def getLRMI(self, *params, response):
try:
lrmi = list(
map(
lambda x: json.loads(x.replace("\r", "").replace("\n", " ")),
response.xpath(self.lrmi_path).getall(),
)
)
except:
logging.warning(
"failed parsing lrmi at " + response.url + ", please check source"
)
return None
for l in lrmi:
value = JSONBase.get(self, *params, json=l)
if value != None:
return html.unescape(value)
return None
def get(self, *params, response):
data = json.loads(response.body_as_unicode())
return JSONBase.get(self, *params, json=data)