def addProcess(self, pid, is_attached, parent=None, is_thread=False):
"""
Add a new process using its identifier. Use is_attached=False to
attach an existing (running) process, and is_attached=True to trace
a new (stopped) process.
"""
if pid in self.dict:
raise KeyError("The process %s is already registered!" % pid)
process = PtraceProcess(self,
pid,
is_attached,
parent=parent,
is_thread=is_thread)
info("Attach %s to debugger" % process)
self.dict[pid] = process
self.list.append(process)
try:
process.waitSignals(SIGTRAP, SIGSTOP)
except KeyboardInterrupt:
error("User interrupt! Force the process %s attach "
"(don't wait for signals)." % pid)
except ProcessSignal as event:
event.display()
except: # noqa: E722
process.is_attached = False
process.detach()
raise
if HAS_PTRACE_EVENTS and self.options:
process.setoptions(self.options)
return process
def readBytes(self, address, size):
if not self.read_mem_file:
filename = '/proc/%u/mem' % self.pid
try:
self.read_mem_file = open(filename, 'rb', 0)
except IOError as err:
message = "Unable to open %s: fallback to ptrace implementation" % filename
if err.errno != EACCES:
error(message)
else:
info(message)
self.readBytes = self._readBytes
return self.readBytes(address, size)
try:
mem = self.read_mem_file
mem.seek(address)
data = mem.read(size)
except (IOError, ValueError) as err:
raise ProcessError(
self, "readBytes(%s, %s) error: %s" %
(formatAddress(address), size, err))
if len(data) == 0 and size:
# Issue #10: If the process was not created by the debugger
# (ex: fork), the kernel may deny reading private mappings of
# /proc/pid/mem to the debugger, depending on the kernel
# version and kernel config (ex: SELinux enabled or not).
#
# Fallback to PTRACE_PEEKTEXT. It is slower but a debugger
# tracing the process is always allowed to use it.
self.readBytes = self._readBytes
return self.readBytes(address, size)
return data
def analyseurl(urls):
"""
功能：分析urls,返回列表格式的字典
字典格式：{'name':names,'urls':url}
这里将符合要求的页面信息插入数据库,还包括日志信息
还包括 key的判断？？？？
mm = re.compile('''\<a.*?href\=['|"](http\w*?)['|"].*?\>''')
"""
returns = []
html = urllib2.urlopen(urls, timeout=50)
#print urls
#try:
if True:
data = html.read()
#soup = BeautifulSoup.BeautifulSoup(data)
#temp = soup.findAll('a',href=re.compile(r'http.*?\W'))#为什么不直接用re匹配a标签,使用beautifulsoup只能匹配出15个，怎么回事呢
mm = re.compile('''\<a\W*?href\="(http.*?)".*?\>''')
temp = mm.findall(data)
logging2.debug('analysing ' + urls)
#print 'analysing'
for tt in temp:
returns.append(tt)
conn = sqlite3.connect(options.dbfile)
cor = conn.cursor()
cor.execute(
'create table if not exists keyofhtml( id integer primary key,urls text,key text,htmls text)'
)
#print 0,'0'
rr = re.compile(
r"""content\W*?\=\W*?["|']\W*?text\/html\W*?\;\W*?charset\W*?\=\W*?(\w*?)\W*?["|']"""
)
m = rr.search(data)
#print 1,'1'
if m:
#print 2
code = m.group(1)
try:
data = data.decode(code)
except UnicodeDecodeError, e:
#print e
logging2.error('decode from charset error')
#print 4
rekey = re.compile(keyinsys) #生成关键字匹配
good = rekey.search(data)
if good:
#print 'good'
data = data.replace("'", '"') #纠结的单引号怎么处理？
sqls = "insert into keyofhtml(urls,key,htmls) values('%s','%s','%s')"
try:
cor.execute(sqls % (urls, keyinsys, data))
except UnicodeDecodeError, e:
#print e
cor.execute(sqls % (urls, keyinsys, 'decode error'))
logging2.error('reading ' + urls + ' decode error')
conn.commit()
def processExited(self, event):
# Display syscall which has not exited
state = event.process.syscall_state
if (state.next_event == "exit") \
and (not self.options.enter) \
and state.syscall:
self.displaySyscall(state.syscall)
# Display exit message
error("*** %s ***" % event)
def setupDebugger(self):
# Set ptrace options
if self.options.fork:
try:
self.debugger.traceFork()
except DebuggerError:
error("--fork option is not supported by your OS, sorry!")
exit(1)
if self.options.trace_exec:
self.debugger.traceExec()
def analyseurl(urls):
"""
功能：分析urls,返回列表格式的字典
字典格式：{'name':names,'urls':url}
这里将符合要求的页面信息插入数据库,还包括日志信息
还包括 key的判断？？？？
"""
returns = []
print urls
html = urllib2.urlopen(urls, timeout=50)
try:
conn = sqlite3.connect(options.dbfile)
cor = conn.cursor()
cor.execute(
'create table if not exists keyofhtml( id integer primary key,urls text,key text,htmls text)'
)
data = html.read()
rr = re.compile(r"""content\=["|']text\/html\;charset\=(\w*?)["|']""")
m = rr.search(data)
if m:
code = m.group(1)
if code:
data = data.decode(code)
rekey = re.compile(keyinsys)
good = rekey.search(data)
if good:
data = data.replace("'", '"') #纠结的单引号怎么处理？
sqls = "insert into keyofhtml(urls,key,htmls) values('%s','%s','%s')"
cor.execute(sqls % (urls, keyinsys, data))
conn.commit()
conn.close()
logging2.debug('reading ' + urls)
logging2.info('what should i write here')
logging2.warning('a warning here')
logging2.error('a error test here')
logging2.critical('what is a critical??')
#print 'reading'
except:
print 'error'
logging2.error('error ong reading ' + urls)
soup = BeautifulSoup.BeautifulSoup(data)
temp = soup.findAll('a', href=re.compile(r'http.*')) #为什么不直接用re匹配a标签
logging2.debug('analysing ' + urls)
#print 'analysing'
for tt in temp:
hrefs = tt['href'] #have?
if hrefs.startswith('http'):
if tt.string: #span?????
returns.append({'name': tt.string, 'urls': hrefs})
else:
returns.append({'name': 'NoName', 'urls': hrefs})
else:
continue
return returns
def analyseurl(urls):
"""
功能：分析urls,返回列表格式的字典
字典格式：{'name':names,'urls':url}
这里将符合要求的页面信息插入数据库,还包括日志信息
还包括 key的判断？？？？
mm = re.compile('''\<a.*?href\=['|"](http\w*?)['|"].*?\>''')
"""
returns=[]
html = urllib2.urlopen(urls,timeout=50)
#print urls
#try:
if True:
data = html.read()
#soup = BeautifulSoup.BeautifulSoup(data)
#temp = soup.findAll('a',href=re.compile(r'http.*?\W'))#为什么不直接用re匹配a标签,使用beautifulsoup只能匹配出15个，怎么回事呢
mm = re.compile('''\<a\W*?href\="(http.*?)".*?\>''')
temp = mm.findall(data)
logging2.debug('analysing '+urls)
#print 'analysing'
for tt in temp:
returns.append({'urls':tt})
conn = sqlite3.connect(options.dbfile)
cor = conn.cursor()
cor.execute('create table if not exists keyofhtml( id integer primary key,urls text,key text,htmls text)')
#print 0,'0'
rr = re.compile(r"""content\W*?\=\W*?["|']\W*?text\/html\W*?\;\W*?charset\W*?\=\W*?(\w*?)\W*?["|']""")
m = rr.search(data)
#print 1,'1'
if m:
#print 2
code = m.group(1)
try:
data = data.decode(code)
except UnicodeDecodeError,e:
#print e
logging2.error('decode from charset error')
#print 4
rekey = re.compile('.*')
good = rekey.search(data)
if good:
#print 'good'
data = data.replace("'",'"')#纠结的单引号怎么处理？
sqls = "insert into keyofhtml(urls,key,htmls) values('%s','%s','%s')"
try:
cor.execute(sqls%(urls,keyinsys,data))
except UnicodeDecodeError,e:
#print e
cor.execute(sqls%(urls,keyinsys,'decode error'))
logging2.error('reading '+urls+' decode error')
conn.commit()
def displaySyscall(self, syscall):
text = syscall.format()
if syscall.result is not None:
text = "%-40s = %s" % (text, syscall.result_text)
prefix = []
if self.options.show_pid:
prefix.append("[%s]" % syscall.process.pid)
if self.options.show_ip:
prefix.append("[%s]" % formatAddress(syscall.instr_pointer))
if prefix:
text = ''.join(prefix) + ' ' + text
error(text)
def analyseurl(urls):
"""
功能：分析urls,返回列表格式的字典
字典格式：{'name':names,'urls':url}
这里将符合要求的页面信息插入数据库,还包括日志信息
还包括 key的判断？？？？
"""
returns=[]
print urls
html = urllib2.urlopen(urls,timeout=50)
try:
conn = sqlite3.connect(options.dbfile)
cor = conn.cursor()
cor.execute('create table if not exists keyofhtml( id integer primary key,urls text,key text,htmls text)')
data = html.read()
rr = re.compile(r"""content\=["|']text\/html\;charset\=(\w*?)["|']""")
m = rr.search(data)
if m:
code = m.group(1)
if code:
data = data.decode(code)
rekey = re.compile(keyinsys)
good = rekey.search(data)
if good:
data = data.replace("'",'"')#纠结的单引号怎么处理？
sqls = "insert into keyofhtml(urls,key,htmls) values('%s','%s','%s')"
cor.execute(sqls%(urls,keyinsys,data))
conn.commit()
conn.close()
logging2.debug('reading '+urls)
logging2.info('what should i write here')
logging2.warning('a warning here')
logging2.error('a error test here')
logging2.critical('what is a critical??')
#print 'reading'
except:
print 'error'
logging2.error('error ong reading '+urls)
soup = BeautifulSoup.BeautifulSoup(data)
temp = soup.findAll('a',href=re.compile(r'http.*'))#为什么不直接用re匹配a标签
logging2.debug('analysing '+urls)
#print 'analysing'
for tt in temp:
hrefs = tt['href']#have?
if hrefs.startswith('http'):
if tt.string:#span?????
returns.append({'name':tt.string,'urls':hrefs})
else:
returns.append({'name':'NoName','urls':hrefs})
else:
continue
return returns
def _main(self):
self.debugger = PtraceDebugger()
try:
self.runDebugger()
except ProcessExit as event:
self.processExited(event)
except PtraceError as err:
error("ptrace() error: %s" % err)
except KeyboardInterrupt:
error("Interrupted.")
except PTRACE_ERRORS as err:
writeError(getLogger(), err, "Debugger error")
self.debugger.quit()
def analyseurl(urls):
"""
功能：分析urls,返回列表格式的字典
字典格式：{'name':names,'urls':url}
这里将符合要求的页面信息插入数据库,还包括日志信息
"""
returns = []
#print urls
html = urllib2.urlopen(urls, timeout=30)
try:
data = html.read()
rr = re.compile(r"""content\=["|']text\/html\;charset\=(\w*?)["|']""")
m = rr.search(data)
if m:
code = m.group(1)
if code:
data = data.decode(code)
logging2.debug('reading')
#print 'reading'
except:
logging2.error('error ong reading')
soup = BeautifulSoup.BeautifulSoup(data)
temp = soup.findAll('a', href=re.compile(r'http.*'))
logging2.debug('analysing')
#print 'analysing'
for tt in temp:
hrefs = tt['href'] #have?
if hrefs.startswith('http'):
if tt.string: #span?????
returns.append({'name': tt.string, 'urls': hrefs})
else:
returns.append({'name': 'NoName', 'urls': hrefs})
else:
continue
return returns
def analyseurl(urls):
"""
功能：分析urls,返回列表格式的字典
字典格式：{'name':names,'urls':url}
这里将符合要求的页面信息插入数据库,还包括日志信息
"""
returns=[]
#print urls
html = urllib2.urlopen(urls,timeout=30)
try:
data = html.read()
rr = re.compile(r"""content\=["|']text\/html\;charset\=(\w*?)["|']""")
m = rr.search(data)
if m:
code = m.group(1)
if code:
data = data.decode(code)
logging2.debug('reading')
#print 'reading'
except:
logging2.error('error ong reading')
soup = BeautifulSoup.BeautifulSoup(data)
temp = soup.findAll('a',href=re.compile(r'http.*'))
logging2.debug('analysing')
#print 'analysing'
for tt in temp:
hrefs = tt['href']#have?
if hrefs.startswith('http'):
if tt.string:#span?????
returns.append({'name':tt.string,'urls':hrefs})
else:
returns.append({'name':'NoName','urls':hrefs})
else:
continue
return returns
def createProcess(self):
if self.options.pid:
pid = self.options.pid
is_attached = False
error("Attach process %s" % pid)
else:
pid = self.createChild(self.program)
is_attached = True
try:
return self.debugger.addProcess(pid, is_attached=is_attached)
except (ProcessExit, PtraceError) as err:
if isinstance(err, PtraceError) \
and err.errno == EPERM:
error(
"ERROR: You are not allowed to trace process %s (permission denied or process already traced)" % pid)
else:
error("ERROR: Process can no be attached! %s" % err)
return None
def newProcess(self, event):
process = event.process
error("*** New process %s ***" % process.pid)
self.prepareProcess(process)
process.parent.syscall()
def processExecution(self, event):
process = event.process
error("*** Process %s execution ***" % process.pid)
process.syscall()
#print 'good'
data = data.replace("'",'"')#纠结的单引号怎么处理？
sqls = "insert into keyofhtml(urls,key,htmls) values('%s','%s','%s')"
try:
cor.execute(sqls%(urls,keyinsys,data))
except UnicodeDecodeError,e:
#print e
cor.execute(sqls%(urls,keyinsys,'decode error'))
logging2.error('reading '+urls+' decode error')
conn.commit()
#print 'donessss'
conn.close()
logging2.debug('reading '+urls)
logging2.info('what should i write here')
logging2.warning('a warning here')
logging2.error('a error test here')
logging2.critical('what is a critical??')
#print 'reading'
#except:
#print 'error'
#logging2.error('error ong reading '+urls)
return returns
def main():
i = 0
th = threading2.ThreadPool(workQueue,resultQueue,options.number)
td = threading2.MyThread2(workQueue,resultQueue,i,10)#屏幕打印进程
while i <= options.deep:#层次循环
def createChild(self, program):
pid = Application.createChild(self, program)
error("execve(%s, %s, [/* 40 vars */]) = %s" %
(program[0], program, pid))
return pid
#print 'good'
data = data.replace("'", '"') #纠结的单引号怎么处理？
sqls = "insert into keyofhtml(urls,key,htmls) values('%s','%s','%s')"
try:
cor.execute(sqls % (urls, keyinsys, data))
except UnicodeDecodeError, e:
#print e
cor.execute(sqls % (urls, keyinsys, 'decode error'))
logging2.error('reading ' + urls + ' decode error')
conn.commit()
#print 'donessss'
conn.close()
logging2.debug('reading ' + urls)
logging2.info('what should i write here')
logging2.warning('a warning here')
logging2.error('a error test here')
logging2.critical('what is a critical??')
return returns
def main():
"""
执行入口，层次判断，任务转移.
>>> main()
时间 深度 当前完成 待完成
"""
i = 0
th = threading2.ThreadPool(workQueue, resultQueue, options.number)
td = threading2.MyThread2(workQueue, resultQueue, i, 10) #屏幕打印进程