def scrape(self):
url_vars = {
'agent' : self.s_agent,
'referer' : self.s_referer,
'cookie' : self.s_cookie,
'method' : self.s_method,
'postdata' : self.s_postdata,
'headers' : self.s_headers,
}
for i in xrange(5):
try: del vars(self)["".join(['v', str(i+1)])]
except: pass
self._printv(2, 'Scrape - ' + self.s_url)
self._printv(2, 'Regex - ' + self.regex)
self._printv(2, 'Params - ' + str(url_vars))
rawdata = urlopen(self.__app__, self.s_url, url_vars)
if self.s_action == 'read':
self.htmRaw = rawdata['content'].read()
rawdata['content'].close()
self._match(self.htmRaw)
elif self.s_action == 'headers':
self.htmRaw = ''
elif self.s_action == 'geturl':
self.htmRaw = ''
self.v1 = rawdata['geturl']
self.cookies = rawdata['cookies']
self.headers = rawdata['headers']
self.geturl = rawdata['geturl']
def report(self):
vars = [
"".join([key, '=', quote_plus(item)])
for (key, item) in self.__matchresults__.items()
]
vars.append("".join(['phase=', str(self.phase + 1)]))
url = "".join([self.__item__['processor'], '?', "&".join(vars)])
self._printv(0, 'Report with url= ' + url)
rawdata = urlopen(
self.__app__, str(url), {
'cookie':
'version=' + str(self.__app__.navi_version) + '.' +
str(self.__app__.navi_sub_version) + '; platform=' +
self.__app__.os
})
htmRaw = rawdata['content'].read()
htmRaw = re.sub('(?m)\r[#].+|\n[#].+|^\s+|\s+$', '\r\n',
htmRaw) #remove comments and tabs
htmRaw = re.sub('[\r\n]+', '\n', htmRaw) #remove empty lines
datalist = htmRaw.replace('\t', '').split('\n')
rawdata['content'].close()
self._pass = True
nipl = NIPL(self.__app__, self.__item__, self.phase + 1, datalist,
LogDebug)
return nipl.process()
def report(self):
vars = ["".join([key, "=", quote_plus(item)]) for (key, item) in self.__matchresults__.items()]
vars.append("".join(["phase=", str(self.phase + 1)]))
url = "".join([self.__item__["processor"], "?", "&".join(vars)])
self._printv(0, "Report with url= " + url)
rawdata = urlopen(
self.__app__,
str(url),
{
"cookie": "version="
+ str(self.__app__.navi_version)
+ "."
+ str(self.__app__.navi_sub_version)
+ "; platform="
+ self.__app__.os
},
)
htmRaw = rawdata["content"].read()
htmRaw = re.sub("(?m)\r[#].+|\n[#].+|^\s+|\s+$", "\r\n", htmRaw) # remove comments and tabs
htmRaw = re.sub("[\r\n]+", "\n", htmRaw) # remove empty lines
datalist = htmRaw.replace("\t", "").split("\n")
rawdata["content"].close()
self._pass = True
nipl = NIPL(self.__app__, self.__item__, self.phase + 1, datalist, LogDebug)
return nipl.process()
def report(self):
vars = ["".join([key,'=',quote_plus(item)]) for (key, item) in self.__matchresults__.items()]
vars.append("".join(['phase=', str(self.phase+1)]))
url = "".join([self.__item__['processor'], '?', "&".join(vars) ])
self._printv(0, 'Report with url= '+url)
rawdata = urlopen(self.__app__, str(url), {'cookie':'version='+str(self.__app__.navi_version)+'.'+str(self.__app__.navi_sub_version)+'; platform='+self.__app__.os})
htmRaw = rawdata['content'].read()
htmRaw = re.sub('(?m)\r[#].+|\n[#].+|^\s+|\s+$', '\r\n', htmRaw) #remove comments and tabs
htmRaw = re.sub('[\r\n]+', '\n', htmRaw) #remove empty lines
datalist = htmRaw.replace('\t','').split('\n')
rawdata['content'].close()
self._pass = True
nipl = NIPL(self.__app__, self.__item__, self.phase+1, datalist, LogDebug)
return nipl.process()
def scrape(self):
url_vars = {
"agent": self.s_agent,
"referer": self.s_referer,
"cookie": self.s_cookie,
"method": self.s_method,
"postdata": self.s_postdata,
"headers": self.s_headers,
}
for i in xrange(5):
try:
del vars(self)["".join(["v", str(i + 1)])]
except:
pass
self._printv(2, "Scrape - " + self.s_url)
self._printv(2, "Regex - " + self.regex)
self._printv(2, "Params - " + str(url_vars))
rawdata = urlopen(self.__app__, self.s_url, url_vars)
if self.s_action == "read":
self.htmRaw = rawdata["content"].read()
rawdata["content"].close()
self._printv(1, "Scrape htmRaw - " + self.htmRaw)
self._match(self.htmRaw)
elif self.s_action == "headers":
self.htmRaw = ""
elif self.s_action == "geturl":
self.htmRaw = ""
self.v1 = rawdata["geturl"]
self.cookies = rawdata["cookies"]
self.headers = rawdata["headers"]
self.geturl = rawdata["geturl"]
