def run(self, ip, port=25):
try:
s = socket.socket()
s.connect((ip, port))
banner = s.recv(1024).rstrip('\r\n')
if banner[:3] == '220':
self.data.banner = banner[4:]
s.send('EHLO test\n')
ehlo = s.recv(1024).replace('\r', '').rstrip().split('\n')[1:]
self.data.ehlo = map(lambda x: x[4:], ehlo)
except Exception as e:
cprint(str(e), 'error')
self.clear()
s.close()
return None
if 'STARTTLS' in self.data.ehlo:
try:
s.send("STARTTLS\n")
resp = s.recv(1024)
if resp[:3] == '220':
ss = ssl.wrap_socket(s)
cert = ss.getpeercert(True)
self.data.update(parse_der(cert))
except Exception as e:
cprint(str(e), 'error')
self.clear()
return None
finally:
s.close()
ServicesInfo.add(ip, port, 'smtp', self.data)
self.clear()
return True
def nouse_run(self, ipdir):
"""
A wrapper for single_run_nmap.
:param ipdir: dict. Using ip:ports as key:value.
:return: dict. Further imformation about services on each host.
"""
result = {}
for ip in ipdir:
tmp_res = self.single_run_nmap(ip, ipdir[ip])
if tmp_res != None:
result[ip] = tmp_res
if self.error_list:
cprint(
'Scanning finished, but some error occurs while nmap scanning!',
'error')
with open('test.log', 'w') as f:
for error in self.error_list:
f.write("{}\n".format(error))
else:
cprint('One ip range scanning finished with no error!', 'info')
self.nmap_result = result
NmapInfo.addWithJson(dumps(result))
def single_run_zmap(self, port, ips):
"""
Must run this function as root. Otherwise fail.
:param ports: str. the port that zmap scans
:param ips: str. ip addresses range in CIDR block notation
"""
cprint("Run zmap on port {} of {}".format(port, ips), 'debug')
os.seteuid(0) # run as root.
if not self.zmap_path:
print 'Zmap is not installed!'
exit()
cmd = [self.zmap_path] + config.common.ZMAP_CMD + ["-p", port, ips]
cprint("start process: " + str(cmd), "debug")
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = p.communicate()
if err:
cprint(err, 'error')
return
if out:
cprint("One zmap scanning finished!", "debug")
else:
cprint("No result found this time!", "warning")
return
for ip in out.strip().split('\n'):
if self.zmap_result.has_key(ip):
self.zmap_result[ip].append(port)
else:
self.zmap_result[ip] = [port]
def _generate_all_tasks_for_goal(self, goal, max_depth=3):
base_entities = [goal]
intermediate_entities = set()
cprint(DEBUG, f"Expanding tasks to goal {goal}")
self._expand_tasks_to_goal(goal, max_depth, base_entities,
intermediate_entities)
cprint(DEBUG, "Done.")
def run(self, ip, port=25):
try:
s = socket.socket()
s.connect((ip, port))
banner = s.recv(1024).rstrip('\r\n')
if banner[:3] == '220':
self.data.banner = banner[4:]
s.send('EHLO test\n')
ehlo = s.recv(1024).replace('\r', '').rstrip().split('\n')[1:]
self.data.ehlo = map(lambda x: x[4:], ehlo)
except Exception as e:
cprint(str(e), 'error')
self.clear()
return None
if 'STARTTLS' in self.data.ehlo:
try:
s.send("STARTTLS\n")
resp = s.recv(1024)
if resp[:3] == '220':
ss = ssl.wrap_socket(s)
cert = ss.getpeercert(True)
self.data.update(parse_der(cert))
except Exception as e:
cprint(str(e), 'error')
self.clear()
return None
finally:
s.close()
ServicesInfo.add(ip, port, 'smtp', self.data)
self.clear()
return True
def run(self, ip, ports=config.common.PORTS):
"""
Must run as root.
Use nmap to get further imformation about the ip and ports
scanned by zmap.
:param ip: str. The target ip that zmap will scan.
:param port: list. All open ports of the ip founded by zmap.
:return: list. Further imformation about services on the host.
"""
self.current_ip = ip
filename = os.path.join(
config.paths.logpath,
'{}_{}.xml.log'.format(ip, '-'.join(map(str, ports))))
# Save time while debugging by not scanning ip that has been scanned.
if 0 and os.path.exists(filename):
cprint(
'Previous scan result exists for {}, just parse the xml.'.
format(ip), 'info')
return self.parse_nmap_xml(filename)
# os.seteuid(0) # run as root.
cprint(
'Start scanning {} on port {} with nmap'.format(
ip, ','.join(ports)), 'info')
cmd = [self.nmap_path] + config.common.NMAP_CMD + \
[ip, '-p', ','.join(ports)]
v6Scan = False
if not isIPv4(ip): # ipv6
cmd.append('-6')
v6Scan = True
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, error = p.communicate()
if error:
cprint(error, 'error')
cprint(
'Nmap finished scanning {} on {}'.format(ip,
','.join(map(str,
ports))),
'info')
# store temporary xml file in logs dir.
f = open(filename, 'w+')
f.write(out)
f.close()
cprint('nmap result %s' % self.parse_nmap_xml(filename), 'debug')
self.nmap_result[ip] = self.parse_nmap_xml(filename)
if v6Scan:
for port in self.nmap_result[ip]:
ServicesInfo.add(
ip, port, self.nmap_result[ip][port]["name"], {
"version": self.nmap_result[ip][port]["version"],
"extrainfo": self.nmap_result[ip][port]["extrainfo"],
"product": self.nmap_result[ip][port]["product"],
})
else:
NmapInfo.addWithJson(dumps(self.nmap_result))
def run(self, ip, port=110):
try:
s = socket.socket()
s.connect((ip, port))
banner = s.recv(1024).rstrip('\r\n')
self.data.banner = banner
if banner.startswith('+OK'):
s.send('CAPA\r\n')
capa = s.recv(1024).split('\r\n')[1:-2]
self.data.capability_list = capa
except Exception as e:
cprint(str(e), 'error')
self.clear()
return None
if 'STLS' in self.data.capability_list:
try:
s.send("STLS\r\n")
resp = s.recv(1024)
if resp[:3] == '+OK':
ss = ssl.wrap_socket(s)
cert = ss.getpeercert(True)
self.data.update(parse_der(cert))
except Exception as e:
cprint(str(e), 'error')
self.clear()
return None
s.close()
# from pprint import pprint
# pprint(self.data)
ServicesInfo.add(ip, port, 'pop3', self.data)
self.clear()
return True
def run(self, ip, port=3306):
try:
s = socket.socket()
s.connect((ip, port))
s.recv(4)
self.data.protocol_version = ord(s.recv(1))
self.data.server_version = recv_str(s)
self.data.connection_id = unpack('I', s.recv(4))[0]
self.data.auth_data = recv_str(s)
if self.data.protocol_version == 10:
cflag1 = s.recv(2)
self.data.character_set = ord(s.recv(1))
self.data.status_flags = unpack('H', s.recv(2))[0]
self.data.capability_flags = unpack('I', cflag1+s.recv(2))[0]
auth_len = ord(s.recv(1))
assert set(s.recv(10)) == set(['\x00'])
self.data.auth_data += recv_str(s)
if auth_len:
self.data.auth_name = s.recv(auth_len)
assert s.recv(1) == '\x00'
except Exception as e:
cprint(str(e), 'error')
self.clear()
return None
ServicesInfo.add(ip, port, 'mysql', self.data)
s.close()
self.clear()
return True
def single_run_zmap(self, port, ips):
"""
Must run this function as root. Otherwise fail.
:param ports: str. the port that zmap scans
:param ips: str. ip addresses range in CIDR block notation
"""
cprint("Run zmap on port {} of {}".format(port, ips), 'debug')
os.seteuid(0) # run as root.
if not self.zmap_path:
print 'Zmap is not installed!'
exit()
cmd = [self.zmap_path] + config.common.ZMAP_CMD + ["-p", port, ips]
cprint("start process: " + str(cmd), "debug")
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate()
if err:
cprint(err, 'error')
return
if out:
cprint("One zmap scanning finished!", "debug")
else:
cprint("No result found this time!", "warning")
return
for ip in out.strip().split('\n'):
if self.zmap_result.has_key(ip):
self.zmap_result[ip].append(port)
else:
self.zmap_result[ip] = [port]
def __init__(self):
"""
Initialize the zmap scanner.
"""
super(ZmapScan, self).__init__()
self.zmap_result = {}
self.zmap_path = subprocess.check_output(['which', 'zmap']).rstrip('\n')
if not self.zmap_path:
cprint('Zmap is not installed!', 'error')
exit()
def __init__(self):
"""
Initialize the zmap scanner.
"""
super(ZmapScan, self).__init__()
self.zmap_result = {}
self.zmap_path = subprocess.check_output(['which',
'zmap']).rstrip('\n')
if not self.zmap_path:
cprint('Zmap is not installed!', 'error')
exit()
def __init__(self):
"""
Initialize the nmap scanner.
"""
super(NmapScan, self).__init__()
self.nmap_path = subprocess.check_output(['which', 'nmap']).rstrip('\n')
if not self.nmap_path:
cprint('Nmap is not installed!', 'error')
exit()
self.error_list = []
self.nmap_result = {}
self.current_ip = None
def run(self, ips, ports=config.common.PORTS):
"""
A wrapper for single_run_zmap.
:param ports: list. the ports that zmap scans
:param ips: str. ip addresses range in CIDR block notation
"""
cprint("Start running zmap on port {} of {}".format(''.join(ports), ips), 'info')
for port in ports:
self.single_run_zmap(port, ips)
cprint("All zmap scanning finished!", "info")
ZmapInfo.addWithJson(dumps(self.zmap_result))
def __init__(self):
"""
Initialize the nmap scanner.
"""
super(NmapScan, self).__init__()
self.nmap_path = subprocess.check_output(['which',
'nmap']).rstrip('\n')
if not self.nmap_path:
cprint('Nmap is not installed!', 'error')
exit()
self.error_list = []
self.nmap_result = {}
self.current_ip = None
def run(self, ips, ports=config.common.PORTS):
"""
A wrapper for single_run_zmap.
:param ports: list. the ports that zmap scans
:param ips: str. ip addresses range in CIDR block notation
"""
cprint(
"Start running zmap on port {} of {}".format(''.join(ports), ips),
'info')
for port in ports:
self.single_run_zmap(port, ips)
cprint("All zmap scanning finished!", "info")
ZmapInfo.addWithJson(dumps(self.zmap_result))
def run(self, ip, port=21, timeout=2):
try:
ftp = FTP()
ftp.connect(ip, port, timeout=timeout)
self.data.banner = ftp.getwelcome()
ftp.login()
self.data.anonymous = True
flist = []
ftp.retrlines('LIST', lambda i: flist.append(i))
ftp.quit()
self.data.flist = flist
ServicesInfo.add(ip, port, 'ftp', self.data)
except Exception, e:
cprint(str(e), 'error')
return None
def run(self, ip, port=23):
try:
s = socket.socket()
s.connect((ip, port))
banner = s.recv(1024)
if banner.endswith('login: '******'telnet', self.data)
except Exception as e:
cprint(str(e), 'error')
return None
finally:
s.close()
self.clear()
return True
def run(self, ip, port=23):
try:
s = socket.socket()
s.connect((ip, port))
banner = s.recv(1024)
if banner.endswith('login: '******'telnet', self.data)
except Exception as e:
cprint(str(e), 'error')
return None
finally:
s.close()
self.clear()
return True
def _expand_tasks_to_goal(self, goal, max_depth=1, base_entities=[], intermediate_entities=set(), relevant_recipes=[]):
"""
DFS expansion of recipes for an entity to generate new tasks
"""
for b in base_entities:
if b not in self.root_entities: # Can't expand if it's a root entity or cyclic
if b != goal: intermediate_entities.add(b)
next_base_entities = base_entities[:]
next_base_entities.remove(b)
cur_depth = len(intermediate_entities) + 1
cprint(DEBUG,'--Expanding base entity', b)
# Expand each recipe for each base entity
for recipe in self.entity2recipes[b]:
cprint(DEBUG,f'----Trying recipe for {b}, {recipe}')
expanded_entities = [e for e in recipe if e not in next_base_entities]
is_cycle = False
for e in recipe:
if e in intermediate_entities or e == goal:
cprint(DEBUG,f'------Cycle detected, skipping recipe {recipe}')
is_cycle = True
break
if is_cycle:
continue
old_base_entities = next_base_entities
next_base_entities = expanded_entities + next_base_entities
# Add task
relevant_recipes.append(recipe)
task = Task(goal, next_base_entities, intermediate_entities, relevant_recipes[:])
if task not in self.depth2task[cur_depth]:
self.depth2task[cur_depth].add(task)
cprint(DEBUG,f'------Adding task {task}')
if cur_depth < max_depth:
cprint(DEBUG,f'current depth is {cur_depth}')
self._expand_tasks_to_goal(goal, max_depth, next_base_entities, intermediate_entities, relevant_recipes[:])
relevant_recipes.remove(recipe)
next_base_entities = old_base_entities
if b != goal: intermediate_entities.remove(b)
def run(self, ip, ports=config.common.PORTS):
"""
Must run as root.
Use nmap to get further imformation about the ip and ports
scanned by zmap.
:param ip: str. The target ip that zmap will scan.
:param port: list. All open ports of the ip founded by zmap.
:return: list. Further imformation about services on the host.
"""
self.current_ip = ip
filename = os.path.join(config.paths.logpath, '{}_{}.xml.log'.format(ip, '-'.join(map(str, ports))))
# Save time while debugging by not scanning ip that has been scanned.
if 0 and os.path.exists(filename):
cprint('Previous scan result exists for {}, just parse the xml.'.format(ip), 'info')
return self.parse_nmap_xml(filename)
# os.seteuid(0) # run as root.
cprint('Start scanning {} on port {} with nmap'.format(ip, ','.join(ports)), 'info')
cmd = [self.nmap_path] + config.common.NMAP_CMD + \
[ip, '-p', ','.join(ports)]
v6Scan = False
if not isIPv4(ip): # ipv6
cmd.append('-6')
v6Scan = True
p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, error = p.communicate()
if error:
cprint(error, 'error')
cprint('Nmap finished scanning {} on {}'.format(
ip, ','.join(map(str, ports))), 'info')
# store temporary xml file in logs dir.
f = open(filename.replace(":", "_"), 'w+')
f.write(out)
f.close()
cprint('nmap result %s' % self.parse_nmap_xml(filename), 'debug')
self.nmap_result[ip] = self.parse_nmap_xml(filename)
if v6Scan:
for port in self.nmap_result[ip]:
ServicesInfo.add(ip, port, self.nmap_result[ip][port]["name"], self.nmap_result[ip][port])
else:
NmapInfo.addWithJson(dumps(self.nmap_result))
def run(self, ip, port=80, ishttps=False):
# WebPage
port = int(port)
if ishttps:
url = "https://" + ip
if port != 443:
url += ":" + str(port)
else:
url = "http://" + ip
if port != 80:
url += ":" + str(port)
try:
webinfo = WebPage(url).info()
ServicesInfo.add(ip, port, ["http", "https"][int(ishttps)], webinfo)
except Exception as e:
cprint(str(e), 'error')
return None
return True
def run(self, ip, port=21, timeout=2):
try:
ftp = FTP()
ftp.connect(ip, port, timeout=timeout)
self.data.banner = ftp.getwelcome()
ftp.login()
self.data.anonymous = True
flist = []
ftp.retrlines('LIST', lambda i: flist.append(i))
self.data.flist = flist
ServicesInfo.add(ip, port, 'ftp', self.data)
except Exception, e:
cprint(str(e), 'error')
return None
def run(self, ip, port=22, timeout=2):
try:
socket.setdefaulttimeout(timeout)
s = socket.socket()
s.connect((ip, port))
banner = s.recv(50).strip('\r\n').split(' ')
try:
self.data.version = banner[0]
self.data.os = banner[1]
except IndexError:
pass
s.send('{}\r\n'.format(banner[0]))
self._raw_recv = s.recv(2048)
s.close()
self._parse_raw_data()
# use paramiko to get hostkey because of lazyless...
tran = Transport((ip, port))
tran.start_client()
pubkey = tran.get_remote_server_key()
self.data.pubkey_name = pubkey.get_name()
fp = pubkey.get_fingerprint()
self.data.pubkey_fingerprint = ':'.join(
map(lambda x: x.encode('hex'), fp))
ServicesInfo.add(ip, port, 'ssh', self.data)
except Exception as e:
cprint(str(e), 'error')
return None
finally:
tran.close()
self.clear()
return True
def run(self, ip, port=22, timeout=2):
try:
socket.setdefaulttimeout(timeout)
s = socket.socket()
s.connect((ip, port))
banner = s.recv(50).strip('\r\n').split(' ')
try:
self.data.version = banner[0]
self.data.os = banner[1]
except IndexError:
pass
s.send('{}\r\n'.format(banner[0]))
self._raw_recv = s.recv(2048)
s.close()
self._parse_raw_data()
# use paramiko to get hostkey because of lazyless...
tran = Transport((ip, port))
tran.start_client()
pubkey = tran.get_remote_server_key()
self.data.pubkey_name = pubkey.get_name()
fp = pubkey.get_fingerprint()
self.data.pubkey_fingerprint = ':'.join(map(lambda x:x.encode('hex'), fp))
ServicesInfo.add(ip, port, 'ssh', self.data)
except Exception as e:
cprint(str(e), 'error')
return None
finally:
tran.close()
self.clear()
return True
def nouse_run(self, ipdir):
"""
A wrapper for single_run_nmap.
:param ipdir: dict. Using ip:ports as key:value.
:return: dict. Further imformation about services on each host.
"""
result = {}
for ip in ipdir:
tmp_res = self.single_run_nmap(ip, ipdir[ip])
if tmp_res != None:
result[ip] = tmp_res
if self.error_list:
cprint('Scanning finished, but some error occurs while nmap scanning!', 'error')
with open('test.log', 'w') as f:
for error in self.error_list:
f.write("{}\n".format(error))
else:
cprint('One ip range scanning finished with no error!', 'info')
self.nmap_result = result
NmapInfo.addWithJson(dumps(result))
def parse_nmap_xml(self, nmap_xml):
"""
Parse the xml file and extract the imforation of web services.
Service info contains (name, version, extrainfo, product, devicetype, ostype...)
:param nmap_xml: str. The name of the xml file to parse.
:return: dict. Use open port number as key and concrete infomation of the web service as value.
"""
try:
tree = ET.parse(nmap_xml)
root = tree.getroot()
result = {}
filter_flag = True
for port in root.find('host').find('ports').findall('port'):
if port.find('state').get('state') not in ('filtered',
'closed'):
filter_flag = False
if port.find('state').get('state') == 'open':
service = port.find('service').attrib
service.pop('conf')
service.pop('method')
result[port.get('portid')] = service
except Exception as e:
cprint(e, 'error')
self.error_list.append(self.current_ip)
return None
# What if we get nothing from the xml...
if not result:
if filter_flag:
cprint(
'All open ports detected by zmap are actually filtered or closed!',
'warning')
else:
cprint('Failed to parse nmap xml!', 'error')
self.error_list.append(self.current_ip)
return None
else:
cprint('Nmap xml parsed!', 'debug')
return result
def parse_nmap_xml(self, nmap_xml):
"""
Parse the xml file and extract the imforation of web services.
Service info contains (name, version, extrainfo, product, devicetype, ostype...)
:param nmap_xml: str. The name of the xml file to parse.
:return: dict. Use open port number as key and concrete infomation of the web service as value.
"""
try:
tree = ET.parse(nmap_xml)
root = tree.getroot()
result = {}
filter_flag = True
for port in root.find('host').find('ports').findall('port'):
if port.find('state').get('state') not in ('filtered', 'closed'):
filter_flag = False
if port.find('state').get('state') == 'open':
service = port.find('service').attrib
service.pop('conf')
service.pop('method')
result[port.get('portid')] = service
except Exception as e:
cprint(e, 'error')
self.error_list.append(self.current_ip)
return None
# What if we get nothing from the xml...
if not result:
if filter_flag:
cprint('All open ports detected by zmap are actually filtered or closed!', 'warning')
else:
cprint('Failed to parse nmap xml!', 'error')
self.error_list.append(self.current_ip)
return None
else:
cprint('Nmap xml parsed!', 'debug')
return result
def run(self, ip, port=143):
try:
s = socket.socket()
s.connect((ip, port))
banner = s.recv(1024).rstrip('\r\n')
self.data.banner = banner
if banner.startswith('* OK'):
s.send('A1 CAPABILITY\r\n')
capa = s.recv(1024).split('\r\n')[0]
self.data.capability_list = capa
except Exception as e:
cprint(str(e), 'error')
self.clear()
return None
if 'ID ' in self.data.capability_list:
try:
s.send('A2 ID ("test" "test")\r\n')
resp = s.recv(1024)
mid = resp[6:resp.index(')')].split()
iddict = {}
for i in range(0, len(mid), 2):
iddict[mid[i].strip('"')] = mid[i+1].strip('"')
self.data.id = iddict
except Exception as e:
cprint(str(e), 'error')
self.clear()
return None
if 'STARTTLS' in self.data.capability_list:
try:
s.send("A3 STARTTLS\r\n")
resp = s.recv(1024)
if resp[:5] == 'A3 OK':
ss = ssl.wrap_socket(s)
cert = ss.getpeercert(True)
self.data.update(parse_der(cert))
except Exception as e:
cprint(str(e), 'error')
self.clear()
return None
s.close()
# from pprint import pprint
# pprint(self.data)
ServicesInfo.add(ip, port, 'imap', self.data)
self.clear()
return True
