def __init__(self):
Tools.__init__(self)
"""
Dictionary storage method
Use # followed by csv fields to define headers e.g. #host, ip, model, serial number
dictionary format example
dict_db[unique numeric key] = {}
dict_db[unique numeric key]['host'] = 'router-01'
dict_db[unique numeric key]['ip'] = '1.1.1.1'
dict_db[unique numeric key]['model'] = 'cisco 2811'
dictionary search index
search_db['router-01'] = {}
search_db['router-01']['tag'] = 'host'
search_db['router-01']['key'] = unique numeric key
search_db['1.1.1.1'] = {}
search_db['router-01']['tag'] = 'ip'
search_db['router-01']['key'] = unique numeric key
host entries appended to the self index list to provide a way to check for unique entries when adding data
data entries appended to the register if unique, if not the key number is added to the entry i.e. Cisco #123
Written by Peter Rogers
(C) Intelligent Planet 2013
"""
self.verbose = 1
self.space_size = 18
self.index = []
self.index_db = {}
self.register = {}
self.dict_db = {}
self.search_db = {}
def __init__(self):
Tools.__init__(self)
"""
MAC lookup to identify the vendor from the OUI of the mac address
Usage example:
>>> x = mac.Mac()
>>> x.id_mac('0000.0c07.ac03')
('Cisco', 'CISCO SYSTEMS')
Tested with Python ver 2.7.2 on Win7 & Win XP
(c) 2012 - 2014 Intelligent Planet Ltd
"""
if "\\" in os.getcwd():
self.path = os.getcwd() + "\\"
else:
self.path = os.getcwd() + "/"
self.mac_file = self.path + "mac"
self.mac_file_url = "http://anonsvn.wireshark.org/wireshark/trunk/manuf"
self.oui_list = []
try:
self.load_mac()
except:
self.create_mac_file() # download mac oui list and save to file as per the self.mac_file value
self.load_mac()
def __init__(self, verbose=0):
Tools.__init__(self)
"""
Cisco IP Cache Flow parse tool and Db viewer
SrcIf SrcIPaddress DstIf DstIPaddress Pr SrcP DstP Pkts
Fa0/1 10.155.20.122 Se0/0:0.101* 10.7.10.124 01 0000 0000 219
Fa0/1 10.155.20.123 Se0/0:0.101* 10.7.10.124 01 0000 0000 219
Fa0/0 10.182.137.8 Se0/0:0.101* 62.239.26.81 06 C508 07D0 2
[flow_id] = {}
[flow_id][SrcIPaddress]
[flow_id][DstIPaddress]
[flow_id][SrcIf]
[flow_id][DstIf]
[flow_id][Protocol]
[flow_id][SrcPort]
[flow_id][DstPort]
[flow_id][Packets]
Written by Peter Rogers
(C) Intelligent Planet 2013
"""
self.load_file = 'c:/cache_load'
self.limit = 200
self.verbose = verbose
self.cache_dict = {}
self.display_heading = '\nSrcIPaddress SrcPort DstIPaddress DstPort Protocol Packets\n'
def __init__(self, verbose=0):
Tools.__init__(self)
"""
Well known port search tool
downloaded port list from
http://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.csv
format of csv file
Service Name,Port Number,Transport Protocol,Description,Assignee,Contact,Registration Date,Modification Date,Reference,Service Code,Known Unauthorized Uses,Assignment Notes
ftp-data,20,tcp,File Transfer [Default Data],[Jon_Postel],[Jon_Postel],,,,,,
ftp-data,20,udp,File Transfer [Default Data],[Jon_Postel],[Jon_Postel],,,,,,
format of dictionary
[portnum_protocol]['name'] = name
[portnum_protocol]['description'] = description
Tested with Python ver 2.7.2 on Win7 & Win XP
(c) 2012 - 2014 Intelligent Planet Ltd
"""
self.verbose = verbose
self.port_dict = {}
self.path = os.getcwd() + '\\'
self.load_file = self.path + 'ports.csv'
self.load()
def __init__(self, ip="", hostname="", out_dict="", auth_con=""):
Tools.__init__(self)
Vty.__init__(self, hostname, out_dict)
"""
interface to the the telnet library
"""
self.ip = ip
self.hostname = hostname.lower()
self.port = 23
self.auth_con = auth_con
self.user = ""
self.password = ""
self.path = os.getcwd() + "\\"
self.newline = "\r\n"
self.space = " "
self.verbose = 1
self.read_until_timeout = 2
self.timeout = 5
self.sleep = 0.1
# set the values for Cisco as default
self.login_text = "Username: "******"Password: "******"q"
self.banner_list = ["\d+>", "--More--", "\d+#"]
self.more = "--More--"
def __init__(self, verbose=0):
Tools.__init__(self)
"""
Well known port search tool
downloaded port list from
http://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.csv
format of csv file
Service Name,Port Number,Transport Protocol,Description,Assignee,Contact,Registration Date,Modification Date,Reference,Service Code,Known Unauthorized Uses,Assignment Notes
ftp-data,20,tcp,File Transfer [Default Data],[Jon_Postel],[Jon_Postel],,,,,,
ftp-data,20,udp,File Transfer [Default Data],[Jon_Postel],[Jon_Postel],,,,,,
format of dictionary
[portnum_protocol]['name'] = name
[portnum_protocol]['description'] = description
Written by Peter Rogers
(C) Intelligent Planet 2013
"""
self.verbose = verbose
self.port_dict = {}
self.load_file = 'c:/ports.csv'
self.load()
def __init__(self):
Tools.__init__(self)
"""
Cisco show command parse tool and Db viewer
sh ip int brief format is:
['Interface', 'IP-Address', 'OK?', 'Method', 'Status', 'Protocol']
Dict format example is:
self.dict_db[self.hostname] = {}
self.dict_db[self.hostname][command] = {}
self.dict_db[self.hostname][command][interface] = {}
self.dict_db[self.hostname][command][interface][ip-address] = value
"""
self.verbose = 0
### set the path to the log and db files ###
try: self.path = 'H:/crt/sessions/'
except: self.path = 'C:/Program Files/SecureCRT/Sessions/'
self.load_file = self.path + 'log'
self.db_file = self.path + 'show_db'
self.load_dict()
self.classify_dict = {'sh ip int brief': ['Interface', 'IP-Address']}
def __init__(self, verbose=0):
Tools.__init__(self)
"""
X.25 Routing parser for cisco routers
format of router output
r01#sh x25 rou | inc dest|0/0
1 dest ^(0000111111).* xot 10.11.12.13
2 dest ^(0000111112).* xot 10.11.12.14
format of dictionary
key = auto incrementing integer
[key]['router'] = hostname
[key]['route_id'] = route_num
[key]['x25_route'] = destination
[key]['ip_address'] = ip_add
[key]['match'] = match
Written by Peter Rogers
(C) Intelligent Planet 2013
"""
self.verbose = verbose
self.x25_dict = {}
self.route_list = []
self.config = []
self.config_list = []
self.load_file = 'c:/x25'
self.out_file = 'c:/x25_out'
self.display_heading = '\n # X.25 route IPaddress match router \n'
self.load()
def __init__(self, verbose=0):
Tools.__init__(self)
"""
Assigned Internet Protocol Numbers search tool
downloaded port list from
http://www.iana.org/assignments/protocol-numbers/protocol-numbers-1.csv
format of csv file
Decimal,Keyword,Protocol,Reference
0,HOPOPT,IPv6 Hop-by-Hop Option,[RFC2460]
1,ICMP,Internet Control Message,[RFC792]
format of dictionary
[Protocol_num]['name'] = name
[Protocol_num]['description'] = description
Tested with Python ver 2.7.2 on Win7 & Win XP
(c) 2012 - 2014 Intelligent Planet Ltd
"""
self.verbose = verbose
self.protocol_dict = {}
self.path = os.getcwd() + '\\'
self.load_file = self.path + 'protocols.csv'
self.load()
def __init__(self, ip='192.168.1.10', name='test', port=23):
Tools.__init__(self)
"""
Network tool kit
Tested on Win XP with Python 2.7
(c) 2012, 2013 Intelligent Planet Ltd
"""
self.init_net(ip, name, port)
self.timeout = 0.2
self.sleep = 0.1
self.verbose = 1
self.buffers = 1024
self.error = 'fail'
self.web_proxy = '172.19.193.122'
self.port_list = [20, 21, 22, 23, 25, 53, 67, 68, 69, 80, 161, 162, 179, 443, 520, 1719, 1720, 1985, 1998, 2000, 2427, 3389, 5060, 5900, 8080]
#FTP(20, 21), SSH(22), Telnet(23), SMTP(25), DNS(53), DHCP(67, 68), TFTP(69), HTTP(80), SNMP(161, 162), BGP(179), HTTPS(443), RIP(520)
#H.323(1719. 1720), HSRP(1985), XOT(1998), SCCP(2000), MGCP(2427), RDP(3389), SIP(5060), VNC(5900)
self.http_get = 'GET /index.html HTTP/1.1 \r\n'
self.http_host = 'Host: %s \r\n\r\n'
def __init__(self):
Tools.__init__(self)
self.url = 'https://ms.localhome.cn/api/base/china-areas?pageSize=500&existCity=true'
self.citys = None
self.conn = None
self.init_spider()
self.list_url = """https://ms.localhome.cn/api/v2/prod/houses?keyword=%s&pageNumber=%s
&bedNumberGreaterThanEqual=1&roomNumberGreaterThanEqual=1&tenantNumberGreaterThanEqual=1
&startDateGreaterThanEqual=%s 00:00:00&endDateLessThan=%s 00:00:00&pageSize=50&pageNum=%s"""
"""
def __init__(self):
Tools.__init__(self)
self.start_url = 'https://www.zhenguo.com/'
self.locations = "https://www.zhenguo.com/api/phx/cprod/locations?cityId=%s"
self.list_url = "https://www.zhenguo.com/api/phx/cprod/products?dateBegin=%s" \
"&dateEnd=%s&cityPinyin=%s&locationCategory=%s&locationId=%s&pageNow=%s"
self.conn = None
self.citys = None
self.search_unit = None
self.zhenguo_city_q = Queue()
self.init_spider()
def __init__(self):
Tools.__init__(self)
"""
English language lookup tools
Word list from http://www-01.sil.org/Linguistics/wordlists/english/wordlist/wordsEn.txt
(c) 2012, 2013 Intelligent Planet Ltd
"""
self.alphabet = 'abcdefghijklmnopqrstuvwxyz'
self.eng_dict = self.load('c:/python27/wordsEn.txt')
self.freq = self.letter_frequency(self.eng_dict)
def __init__(self):
Tools.__init__(self)
"""
English language lookup tools
Word list from http://www-01.sil.org/Linguistics/wordlists/english/wordlist/wordsEn.txt
(c) 2012, 2013 Intelligent Planet Ltd
"""
self.alphabet = 'abcdefghijklmnopqrstuvwxyz'
self.eng_dict = self.load('c:/python27/wordsEn.txt')
self.freq = self.letter_frequency(self.eng_dict)
def __init__(self, verbose=1, maxq=200):
Net.__init__(self)
Tools.__init__(self)
"""
Multithreaded network tools
"""
self.verbose = verbose
self.maxq = maxq
self.timeout = 0.2 #
self.buffers = 256 #for check_port
def __init__(self, ip, name, port, auth_con=''):
Tools.__init__(self)
self.ip = ip
self.name = name
self.port = port
self.auth_con = auth_con
self.verbose = 1
self.telnet_timeout = 2
self.telnet_cmd_timeout = 5
self.telnet_sleep = 0.1
self.path = os.getcwd() + '\\'
self.ssh_key_file = self.path + 'ssh_host_keys'
def __init__(self):
Tools.__init__(self)
"""
Dictionary storage method
Use # followed by csv fields to define headers e.g. #host, ip, model, serial number
device dict = [key][user specified fields]
Written by Peter Rogers
(C) Intelligent Planet 2013
"""
self.verbose = 1
self.key_txt = 'key'
self.space_size = 18
self.index = []
def __init__(self):
Tools.__init__(self)
"""
Search interface for host, ip and extended information within device db
Start point for Command line networking toolkit
Tested with Python ver 2.7.2 on Win7 & Win XP
(c) 2012 - 2014 Intelligent Planet Ltd
"""
self.verbose = 0
### cli vars ###
self.search_txt = '\n\nsearch >>>'
self.user = getpass.getuser()
### set the path to csv device files ###
try:
self.path = 'H:/crt/sessions/'
cfile = self.path + 'telnet.ini'
open(cfile)
except:
try:
self.path = 'C:/Documents and Settings/' + self.user + '/Application Data/VanDyke/SecureCRT/Config/sessions/'
cfile = self.path + 'telnet.ini'
open(cfile)
except: self.path = os.getcwd() + '\\'
self.log_file = self.path + 'log'
self.sticky = 1 #can be set to 0 to disable level1 cli
self.session_fail_delay = 2
self.session_start_delay = 1
self.total_records = len(dev_con.search_db)
self.total_hosts = len(dev_con.index)
self.batch_con = batch.Batch(dev_con, self)
self.host_list = []
self.dns_suffix = []
self.test_dns_suffix()
self.arp_start()
self.level_0()
def __init__(self, ip, hostname, out_dict='', auth_con=''):
Tools.__init__(self)
Vty.__init__(self, hostname, out_dict)
"""
interface to the paramiko open ssl module
"""
self.ip = ip
self.hostname = hostname.lower()
self.port = 22
self.auth_con = auth_con
self.user = ''
self.password = ''
self.path = os.getcwd() + '\\'
self.debug_file = self.path + self.hostname + '_ssh_debug.log'
self.newline = '\r\n'
self.verbose = 1
def __init__(self, verbose=1, maxq=200):
Tools.__init__(self)
"""
Multithreaded network tools
mt_dns_rlook(ip_list) reverse DNS lookup
mt_dns_look(name_list) normal DNS lookup
mt_port_spmip(ip_list, port) check single port on multiple IP's
mt_port_sipmp(port_list, ip) check multiple port on single IP
mt_ping(ip_list) ping test
mt_trace(ip) traceroute
mt_http(url_list) download http
(c) 2012, 2013 Intelligent Planet Ltd
"""
self.verbose = verbose
self.maxq = maxq
self.timeout = 0.2 #tcp timeout
self.buffers = 256 #for check_port and http download
def __init__(self, ip='192.168.1.18', name='pi', port=22):
Tools.__init__(self)
"""
VTY - Telnet and SSH
Note SSH requires the pycrypto package which requires a C++ compiler for Windows.
Microsoft Visual C++ can be downloaded for free and used for this.
(c) 2012, 2013 Intelligent Planet Ltd
"""
self.ip = ip
self.name = name
self.port = port
self.telnet_timeout = 2
self.telnet_cmd_timeout = 5
self.telnet_sleep = 0.1
self.verbose = 1
self.ssh_key_file = 'c:/python27/ssh_host_keys'
self.auth_pi()
def __init__(self):
Tools.__init__(self)
"""
MAC parse tool and Db viewer
sh mac-add dyn
vlan mac address type learn age ports
* 124 0021.5e97.7b34 dynamic Yes 30 Te1/3
2nd variation
vlan mac address type learn ports
* 112 0023.7d35.1470 dynamic Yes Po2
3rd variation -
vlan mac address type protocols port
201 000c.29e0.cd3e dynamic ip,ipx,assigned,other GigabitEthernet2/9
[mac][host_port_vlan][date]
Tested with Python ver 2.7.2 on Win7 & Win XP
(c) 2012 - 2014 Intelligent Planet Ltd
"""
self.verbose = 0
self.path = os.getcwd() + '\\'
self.load_file = self.path + 'mac_load'
self.db_file = self.path + 'mac_db'
try: #load the mac dict file
file = open(self.db_file, 'rb')
self.mac_dict = pickle.load(file)
except: self.mac_dict = {}
finally: file.close()
self.mac_db_length = len(self.mac_dict)
def __init__(self, hostname, ip_address, path=''):
Tools.__init__(self)
"""
This class is designed to create and use session files for Secure CRT
by cloning and modifying a pre made template session file.
This class will add a new line (1) to a pre-made template file
by catenating self.ip_text and self.ip_address and saving the file
with hostname.ini in the relavent folder
Template files must be prepared manually by making a session of each
connection type, including auto login credentials (if required)
The process to create session template files and target directory's is as follows:
1) Open secure crt and make a new session
2) Go to folder C:\Program Files\SecureCRT\Sessions and open the session file in a text editor
2) Create a blank first line in the file
3) Remove the line containing S:"Hostname"=
4) Save a template file for each connection type i.e. telnet.ini, ssh.ini, ssh2.ini
5) Create output folders for each connection type in C:\Program Files\SecureCRT\Sessions
i.e. C:\Program Files\SecureCRT\Sessions\telnet
C:\Program Files\SecureCRT\Sessions\ssh
C:\Program Files\SecureCRT\Sessions\ssh2
Usage of the class:
connection_type options are [telnet, ssh, ssh2]
new connection types can be added by creating a folder and template file
hostname is used to write the name of the session file
ip_address is inserted into the session file and controls the connectivity
session creation example:
from python shell
>>>import session
>>>x = session.SecureCRT('ssh', 'my_router', '10.1.1.1')
>>>x.make_session()
session usage example:
from windows command prompt
c:\>securecrt /S "\ssh\my_router"
from python shell
>>>x.launch_session()
Tested on Win XP with Python 2.7
(c) 2012 - 2014 Intelligent Planet Ltd
"""
#passed in class values
self.hostname = hostname
self.ip_address = ip_address
self.path = path
#test connection_type
if self.ip_address != '0.0.0.0': self.connection_type = self.test()
else: self.connection_type = ''
#file name and path values
self.file_extension = '.ini'
if not self.path:
self.path = 'C:/Program Files/SecureCRT/Sessions/'
self.default = 1
self.output_path = self.path + self.connection_type + '/'
self.output_file_name = self.output_path + self.hostname + self.file_extension
def __init__(self, verbose=0):
"""
Analyse netflow data and establish baselines according to the ASN# and county of origin, duration, size of flows
netflow reports will load from \netflow\ folder
a load file logging list will record each file opened
new files will be opened according to file age
Format of netflow load file will be csv with the following fields:
RouterAddress,InterfaceIn,Protocol,SourceAddress,SourcePort,DestinationAddress,DestinationPort,TypeOfService,BytesInVolume,BytesInRatePerDuration,BytesInPercentOfTotalTraffic,FlowCount,FlowDuration,PacketsInVolume,PacketsInRatePerDuration,PacketsInPercentOfTotalTraffic
Only specified interesting fileds will be loaded:
Protocol
SourceAddress
SourcePort
DestinationAddress
DestinationPort
BytesInVolume
BytesInRatePerDuration
FlowDuration
PacketsInVolume
PacketsInRatePerDuration
format of dictionary will be based on the fact that this is an inbound flow report that is manually created so each flow will be to the same destination - i.e. 193.127.210.129
['destination_ip'] = {}
[destination_ip]['Protocol'] = {}
[destination_ip][Protocol]['src'] = {}
[destination_ip][Protocol][src] = SourceAddress + '_' + SourcePort
[destination_ip][Protocol][src]['DestinationPort'] = DestinationPort
[destination_ip][Protocol][src]['TotalFlowCount'] = (1 if first entry but will increment for each flow seen for this src addrd and port to destination)
[destination_ip][Protocol][src]['BytesInVolume'] = BytesInVolume + will increment for future matching flows
[destination_ip][Protocol][src]['BytesInRatePerDuration'] = BytesInRatePerDuration + will increment for future matching flows
Etc for remaining fields, they will increment for future matching flows
GeoIP lookups will be perfromed to provide the following fields:
AS Number
AS Org Name
Country Code
Latitude
Longitude
flow [src] fields will be checked for the AS# and arrays will be built for BytesinVolume, FlowDuration, TotalFlowCount & PacketsInRatePerDuration for each AS#
when records are parsed each flow will be first compared to the existing array data and checked for anomolys, then the data will be added.
An AS# dict entry will be created for each AS#
AS = {}
AS[CIDR_Size] = num of possible hosts
AS[src_ip] = [] each src ip will be added to the dict to give a count of unique IP's seen from each AS#
Calculate the difference between Milton Keynes Latitude 52.0175 Longitude -.7896 and the source IP in miles
Shenley = (52.0175, -0.7896)
GeoIP city lookup - 'latitude' and 'longitude'
Get an everage of the source distance in miles for each netflow sample and save to the dict at the end
[destination_ip]['Protocol']['Avg_Distance'] = Avg_Distance
Use BytesInVolume distrubution per ASN to determine how trustworthy the AS# is
Use distance to AS# as another measure of trust
Country reputation can be manually fixed per country - i.e. mark down China, Russia, Nork, USA
Use whitelists for known good AS# & black lists for known bad AS#
Use Avg_BytesPerFlow for the highest distrubution AS# to deterine the expected rates
Tested with Python ver 2.7.2 on Win7
(c) 2012 - 2016 Intelligent Planet Ltd
"""
Tools.__init__(self)
self.verbose = verbose
self.path = 'h:\\backup\\' + 'netflow' + '\\'
self.netflow_dict = {}
self.netflow_dict['load_file_history'] = {}
self.netflow_dict['ASN_Stats'] = {}
self.netflow_dict['ASN_Stats']['Total_TotalFlowCount'] = 0
self.netflow_dict['ASN_Stats']['Total_BytesInVolume'] = 0
self.netflow_dict['ASN_Stats']['Avg_BytesPerFlow'] = 0
self.netflow_dict['ASN_Stats']['Dist_TotalFlowCount'] = []
self.netflow_dict['ASN_Stats']['Dist_BytesInVolume'] = []
self.netflow_dict['ASN_Stats']['Dist_FlowCutOff'] = 0.
self.netflow_dict['ASN_Stats']['Dist_BytesCutOff'] = 0.
self.DistFlowCutOffFactor = 0.0005
self.DistBytesCutOffFactor = 0.000005
self.field_map = {'Protocol':-1, 'SourceAddress':-1, 'SourcePort':-1, 'DestinationAddress':-1, 'DestinationPort':-1, 'BytesInVolume':-1, 'BytesInRatePerDuration':-1, 'FlowDuration':-1, 'PacketsInVolume':-1, 'PacketsInRatePerDuration':-1}
#Set the reference point for distance calculations
self.home_city = (52.0175, -0.7896)
#Use the source filter to enable mapping of outbound flows from this IP / Prefix using text matching only
self.SourceFilter = '193.127.210'
#The threshold where flows above this are recorded in detail
self.TrustThreshold = 200
self.open_db()
def __init__(self):
"""Initialization."""
self.argv_setup()
Tools.__init__(self)
