def addPort(portNo): if (Port.objects.filter(portNo=portNo).exists()): return 0 p = Port(portNo=portNo, lastUser=0, opened=False, history="") # the initial data for the different fields p.save() return 1
def job(self, trap):
host = trap['host']
hostname = getSnmp(host, 'SNMPv2-MIB::sysName.0')
event = trap['snmpTrapOID.0']
ifIndex = trap.get('ifIndex')
ifName = trap.get('ifName', getSnmp(host, 'IF-MIB::ifName.' + ifIndex))
ifAlias = trap.get('ifAlias',
getSnmp(host, 'IF-MIB::ifAlias.' + ifIndex))
if 'No Such Instance currently exists at this OID' in ifAlias:
ifAlias = ''
ifAdminStatus = trap.get(
'ifAdminStatus', getSnmp(host, 'IF-MIB::ifAdminStatus.' + ifIndex))
ifAdminStatus = find_state(ifAdminStatus)
ifOperStatus = trap.get(
'ifOperStatus', getSnmp(host, 'IF-MIB::ifOperStatus.' + ifIndex))
ifOperStatus = find_state(ifOperStatus)
if ifAdminStatus or ifOperStatus:
return Port(host=host,
hostname=hostname,
event=event,
ifIndex=ifIndex,
ifName=ifName,
ifAlias=ifAlias,
ifAdminStatus=ifAdminStatus,
ifOperStatus=ifOperStatus)
else:
return None
async def delete_containers(self):
'''
Deletes the containers in docker and the db records
'''
logging.info('Deleting containers')
client = aiodocker.Docker()
containers = await client.containers.list(all=True)
# Remove the containers first
for container in Container.select():
logging.debug('Deleting %s', container.name)
for item in containers:
if item.id == container.container_id:
await item.delete(v=True, force=True)
# Purge the db. Need to find a more efficient way
# to do this
for container in Container.select():
container.delete_instance()
for port in Port.select():
port.delete_instance()
await client.close()
def build_container_map(self):
'''
Creates dictionaries for fast retrieval of container information by index.
'''
container_map_by_port = {}
container_map_by_ip = {}
container_map_by_name = {}
containers = Container.select()
for container in containers:
container_map_by_name[container.name] = container
end_points = Port.select(Port.number, Port.protocol, Container.ip, Container.name,
Container.container_id, Container.start_delay, Container.start_retry_count,
Container.start_on_create, Container.sub_domain) \
.join(Container)\
.order_by(Port.number)
for end_point in end_points:
port_key = self.get_port_map_key(end_point.number,
end_point.protocol)
container = end_point.container
if port_key not in container_map_by_port:
container_map_by_port[port_key] = {}
ip = utility.Net.ipstr_to_int(container.ip)
container_map_by_port[port_key][ip] = container
if ip not in container_map_by_ip:
container_map_by_ip[ip] = container
return container_map_by_port, container_map_by_ip, container_map_by_name
def create(self, the_parsed_port_xml):
"""
Create a port (interface) type model here.
"""
x = the_parsed_port_xml
port_obj = x.get_interface()
include_header_files_list = x.get_include_header_files()
include_header_serial_files_list = x.get_includes_serial_files()
args_obj_list = x.get_args()
#
port_namespace = port_obj.get_namespace()
port_name = port_obj.get_name()
port_comment = port_obj.get_comment()
port_return_type = port_obj.get_return_type()
port_return_modifier = port_obj.get_return_modifier()
#
port_xml_filename = x.get_xml_filename()
#
# Add port args here...
#
args_list = []
for arg in args_obj_list:
n=arg.get_name()
t=arg.get_type()
m=arg.get_modifier()
s=arg.get_size()
c=arg.get_comment()
args_list.append(Arg.Arg(n,t,m,s,c))
#
# Instance the port here...
#
the_port = Port.Port(None, port_name, None, None, None, None, None, port_xml_filename)
the_port.set(port_namespace, args_list, include_header_files_list, include_header_serial_files_list, port_comment)
the_port.set_return(port_return_type, port_return_modifier)
return the_port
async def setup_images(self, images_cfg, update):
logging.info('Setting up images')
count = 0
image_ports = {}
unique_ports = {}
client = aiodocker.Docker()
existing_images = await client.images.list()
# If updating an existing deployment then create a list of
# existing ports
if update:
existing_ports = Port.select(Port.number).distinct()
for existing_port in existing_ports:
port = ExposedPort(existing_port.number,
'{}/tcp'.format(existing_port.number),
TCP_PROTOCOL)
unique_ports[existing_port.number] = port
# Pull down images defined in the images if they are
# not already present on the local system
for image_cfg in images_cfg:
# Pull down the image if it is not already on the system
await self.pull_image(image_cfg.name, existing_images)
image = await client.images.inspect(image_cfg.name)
if Dictionary.has_attr(image, 'ContainerConfig', 'ExposedPorts'):
count += image_cfg.count
exposed_ports = list(image['ContainerConfig']['ExposedPorts'])
for exposed_port in exposed_ports:
match = re.search('(\d+)\/(tcp|udp)', exposed_port)
if match is not None:
port_num = int(match.group(1))
protocol = match.group(2)
if protocol == TCP_PROTOCOL_TXT:
protocol = TCP_PROTOCOL
elif protocol == UDP_PROTOCOL_TXT:
protocol = UDP_PROTOCOL
port = ExposedPort(port_num, exposed_port, protocol)
if image_cfg.name not in image_ports:
image_ports[image_cfg.name] = []
image_ports[image_cfg.name].append(port)
if port_num not in unique_ports:
unique_ports[port_num] = port
await client.close()
return count, image_ports, unique_ports.values()
def get_context(file_path: str) -> Context:
with open(file_path, 'r') as file:
context_dict = json.load(file)
context = Context(**context_dict)
for i in range(len(context.containers)):
container = context.containers[i] = Container(**context.containers[i])
container.ports = Port(**container.ports)
for i in range(len(context.networks)):
context.networks[i] = Network(**context.networks[i])
return context
def search(request):
search_url = request.GET['q']
r = result.result()
information_gathering.nmap(search_url, r)
information_gathering.whatweb(search_url, r)
target = PortTarget()
target.sys_info = r.operatingsystem
target.ip = search_url
target.save()
for item in r.webinformation:
info = WebInfo()
info.target = target
info.info = item
info.save()
for key in r.portservice:
port = Port()
port.target = target
port.number = key
port.details = r.portservice[key]
port.save()
return render(request, 'peni/detail.html', {'target': target})
def search(request):
search_url = request.GET['q']
r = result.result()
information_gathering.nmap(search_url, r)
information_gathering.whatweb(search_url, r)
target = PortTarget()
target.sys_info = r.operatingsystem
target.ip = search_url
target.save()
for item in r.webinformation:
info = WebInfo()
info.target = target
info.info = item
info.save()
for key in r.portservice:
port = Port()
port.target = target
port.number = key
port.details = r.portservice[key]
port.save()
return render(request, 'peni/detail.html', {'target': target})
def port_generate(number: str, detail: Dict) -> None:
# noinspection PyTypeChecker
port = Port.create_or_update({'number': number, 'details': detail})[0]
print('\t%s: Upserted' % port.number)
def create(self, the_parsed_component_xml, parsed_port_xml_list, parsed_serializable_list):
"""
Create a component model here.
"""
x = the_parsed_component_xml
comp_obj = x.get_component()
comp_port_obj_list = x.get_ports()
comp_command_obj_list = x.get_commands()
comp_channel_obj_list = x.get_channels()
comp_parameter_obj_list = x.get_parameters()
comp_event_obj_list = x.get_events()
comp_internal_interface_obj_list = x.get_internal_interfaces()
#
comp_namespace = comp_obj.get_namespace()
comp_name = comp_obj.get_name()
comp_kind = comp_obj.get_kind()
comp_comment = comp_obj.get_comment()
comp_modeler = comp_obj.get_modeler()
if comp_namespace == None:
comp_full_name = comp_name
else:
comp_full_name = comp_namespace + "::" + comp_name
# get original filename here...
comp_xml_filename = x.get_xml_filename()
#
comp_xml_port_files = x.get_port_type_files()
comp_c_header_files = x.get_header_files()
has_guarded_ports = False
num_async_ports = 0
num_sync_ports = 0 # includes guarded ports
#
#print ("Component: %s"%comp_name)
incl_list = []
#
# Create list of ports with all ports of the component.
#
port_obj_list = []
for port_obj in comp_port_obj_list:
n=port_obj.get_name()
t=port_obj.get_type()
d=port_obj.get_direction()
s=port_obj.get_sync()
r=port_obj.get_role()
if (s == 'sync' or s == 'guarded'):
num_sync_ports += 1
if s == 'async':
num_async_ports += 1
p=port_obj.get_priority()
if s == "guarded":
has_guarded_ports = True
c=port_obj.get_comment()
m=port_obj.get_max_number()
f=port_obj.get_full()
port_obj_list.append(Port.Port(n, t, d, s, p, f, c, max_number=m, role=r ))
command_obj_list = []
for command_obj in comp_command_obj_list:
m=command_obj.get_mnemonic()
o=command_obj.get_opcodes()
s=command_obj.get_sync()
p=command_obj.get_priority()
f=command_obj.get_full()
if s == "guarded":
has_guarded_ports = True
if (s == 'sync' or s == "guarded"):
num_sync_ports += 1
if s == 'async':
num_async_ports += 1
c=command_obj.get_comment()
arg_obj_list = []
for a in command_obj.get_args():
name = a.get_name()
atype = a.get_type()
comment = a.get_comment()
size = a.get_size()
arg_obj_list.append(Arg.Arg(name, atype, None, size, comment))
command_obj_list.append(Command.Command(m, o, arg_obj_list, s, p, c, comp_xml_filename,comp_full_name , component_base_name = comp_name , base_opcode = command_obj.get_base_opcode() , full = f))
channel_obj_list = []
for channel_obj in comp_channel_obj_list:
i=channel_obj.get_ids()
n=channel_obj.get_name()
t=channel_obj.get_type()
s=channel_obj.get_size()
c=channel_obj.get_comment()
a=channel_obj.get_abbrev()
f=channel_obj.get_format_string()
u=channel_obj.get_update()
l=channel_obj.get_limits()
channel_obj_list.append(
Channel.Channel(ids=i,
name=n,
ctype=t,
size=s,
abbrev=a,
format_string=f,
update=u,
limits=l,
comment=c,
xml_filename=comp_xml_filename,
component_name=comp_full_name,
component_base_name = comp_name))
event_obj_list = []
for event_obj in comp_event_obj_list:
i=event_obj.get_ids()
n=event_obj.get_name()
s=event_obj.get_severity()
f=event_obj.get_format_string()
t=event_obj.get_throttle()
c=event_obj.get_comment()
arg_obj_list = []
for a in event_obj.get_args():
name = a.get_name()
atype = a.get_type()
size = a.get_size()
comment = a.get_comment()
arg_obj_list.append(Arg.Arg(name, atype, None, size, comment))
event_obj_list.append(Event.Event(i, n, s, f, t, arg_obj_list, c, comp_xml_filename,comp_full_name , component_base_name = comp_name))
internal_interface_obj_list = []
for internal_interface_obj in comp_internal_interface_obj_list:
# borrow this for check
num_async_ports += 1
n=internal_interface_obj.get_name()
p=internal_interface_obj.get_priority()
f=internal_interface_obj.get_full()
c=internal_interface_obj.get_comment()
arg_obj_list = []
for a in internal_interface_obj.get_args():
name = a.get_name()
atype = a.get_type()
size = a.get_size()
comment = a.get_comment()
arg_obj_list.append(Arg.Arg(name, atype, None, size, comment))
internal_interface_obj_list.append(InternalInterface.InternalInterface(n, p, f , arg_obj_list, c, comp_xml_filename,comp_full_name))
parameter_obj_list = []
for parameter_obj in comp_parameter_obj_list:
i=parameter_obj.get_ids()
n=parameter_obj.get_name()
t=parameter_obj.get_type()
set_ops = parameter_obj.get_set_opcodes()
save_ops = parameter_obj.get_save_opcodes()
d=parameter_obj.get_default()
s=parameter_obj.get_size()
c=parameter_obj.get_comment()
parameter_obj_list.append(Parameter.Parameter(i, n, t, set_ops, save_ops, d, s, c, comp_xml_filename,comp_full_name , base_setop = parameter_obj.get_base_setop() , base_saveop = parameter_obj.get_base_saveop()))
serializable_obj_list = []
for serializable_obj in parsed_serializable_list:
f=serializable_obj.get_xml_filename()
n=serializable_obj.get_name()
ns=serializable_obj.get_namespace()
c=serializable_obj.get_comment()
x=serializable_obj.get_includes()
# shouldn't be c includes
m=serializable_obj.get_members()
t=serializable_obj.get_typeid()
serializable_obj_list.append(Serialize.Serialize(f,n,ns,c,x,None,m,t))
#
# Check here to make sure all the port types in the component XML
# exist in the port XMLs
#
interface_xml_list = [parsed_port_obj.get_interface().get_name() for parsed_port_obj in parsed_port_xml_list]
for port_obj in port_obj_list:
t = port_obj.get_type()
## Skip if special port. (If there role)
## Namespaces for special ports are set above
if (t not in interface_xml_list) and (t.lower() != "serial") and (port_obj.get_role() == None):
PRINT.info("ERROR: Missing port type definition in component XML (name: %s, type: %s)" % (port_obj.get_name(),t))
sys.exit(-1)
#
# Check here to make sure all the port types in the component XML
# exist in the port XMLs
#
# interface_xml_list = [parsed_command_obj.get_interface().get_name() for parsed_command_obj in parsed_command_xml_list]
# print interface_xml_list
# for command_obj in command_obj_list:
# t = command_obj.get_type()
# if (t not in interface_xml_list):
# PRINT.info("ERROR: Missing command type definition in component XML (name: %s, type: %s)" % (command_obj.get_type(),t))
# sys.exit(-1)
# #
# Add port type specifics to port object.
# Specifics are things like: args, includes, etc.
for port_obj in port_obj_list:
for parsed_port_obj in parsed_port_xml_list:
#print "Meta: Name: %s, Type: %s" % (port_obj.get_name(), port_obj.get_type())
#print "Meta: Port Type: %s, Port Interface: %s" % (port_obj.get_type(),parsed_port_obj.get_interface().get_name())
if port_obj.get_type() == parsed_port_obj.get_interface().get_name():
arg_obj_list = []
incl_list = parsed_port_obj.get_include_header_files()
namespace = parsed_port_obj.get_interface().get_namespace()
if_comment = parsed_port_obj.get_interface().get_comment()
return_type = parsed_port_obj.get_interface().get_return_type()
return_modifier = parsed_port_obj.get_interface().get_return_modifier()
for a in parsed_port_obj.get_args():
name = a.get_name()
atype = a.get_type()
comment = a.get_comment()
modifier = a.get_modifier()
size = a.get_size()
arg_obj_list.append(Arg.Arg(name, atype, modifier, size, comment))
port_obj.set(namespace, arg_obj_list, incl_list, None, if_comment)
port_obj.set_return(return_type,return_modifier)
# check some rules
# 1) No return values for async ports
if (port_obj.get_sync() == "async") and (return_type != None):
PRINT.info("ERROR: %s: Port \"%s\" cannot be asynchronous and have a return value" % (the_parsed_component_xml.get_xml_filename(), port_obj.get_name()))
sys.exit(-1)
# 2) Serial ports can't have roles
if (port_obj.get_type() == "Serial") and (port_obj.get_role() != None):
PRINT.info("ERROR: %s: Port \"%s\" cannot have a role and be a serialized port" % (the_parsed_component_xml.get_xml_filename(), port_obj.get_name()))
sys.exit(-1)
# check some component/port rules
# 1) Active or queued need at least one async port/command
if (comp_kind == "active") or (comp_kind == "queued"):
if num_async_ports == 0 and len(parameter_obj_list) == 0:
PRINT.info("ERROR: %s: Active/Queued component \"%s\" needs at least one async port, command, or interface" % (the_parsed_component_xml.get_xml_filename(), comp_name))
sys.exit(-1)
# 2) Queued component needs at least one sync port/command
if comp_kind == "queued":
if num_sync_ports == 0:
PRINT.info("ERROR: %s: Queued component \"%s\" needs at least one sync/guarded port or command" % (the_parsed_component_xml.get_xml_filename(), comp_name))
sys.exit(-1)
#
# Instance the component here...
#
the_component = Component.Component(comp_namespace, comp_name, comp_kind, comp_comment, comp_modeler, port_obj_list, command_obj_list, channel_obj_list, parameter_obj_list, event_obj_list, internal_interface_obj_list, serializable_obj_list, comp_xml_filename)
the_component.set_xml_port_files(comp_xml_port_files)
the_component.set_c_header_files(comp_c_header_files)
if (has_guarded_ports):
the_component.set_has_guarded_ports()
#for p in the_component.get_ports():
# print p.get_name(), p.get_namespace()
# for a in p.get_args():
# print a.get_name(), a.get_type(), a.get_modifier()
return the_component
async def create_containers(self, update):
# Clean up all the containers
if not update:
await self.delete_containers()
# Make sure the defined images are available and config the ports
count, image_ports, unique_ports = await self.setup_images(
self.config.images, update)
addresses, subnet, gateway = self.get_available_address(
self.config.network.hosts)
# Do not recreate the network during an update. docker does not provide a
# way to update the network so during an update it would need to be dropped and
# updated. The network id for any existing containers would need to be updated
if not update:
# Create a network for the containers
await self.create_network(self.config.network.hosts, subnet,
gateway)
# Apply the iptable rules required for the configured images
rules = IPTableRules(self.config)
rules.create(subnet, unique_ports)
logging.info('Creating containers')
# Create a unique list of host names
naming = Naming()
host_names = naming.generate_host_names(self.config.naming, count)
scripts_dir = ''.join([os.getcwd(), SCRIPTS_DIR])
logging.info('Using %s for script directory', scripts_dir)
client = aiodocker.Docker()
for image in self.config.images:
# If the image has no exposed ports then there is no use in
# creating a container
if image.name not in image_ports:
continue
ports = [port.text for port in image_ports[image.name]]
ports = dict.fromkeys(ports, {})
for count in range(image.count):
host_name = host_names.pop()
ip = addresses.pop(0)
mac = utility.Net.generate_mac(ip)
config = {
'Hostname': host_name,
'Image': image.name,
'ExposedPorts': ports,
'MacAddress': mac,
'Env': image.env_variables,
'NetworkingConfig': {
'EndpointsConfig': {
'clab': {
'IPAMConfig': {
'IPv4Address': ip,
},
}
}
}
}
if not image.startup_script is None and len(
image.startup_script) > 0:
config['HostConfig'] = {
'Binds': ['{}:{}'.format(scripts_dir, SCRIPTS_DIR)],
}
logging.debug('Creating container %s:%s', host_name,
image.name)
container = await client.containers.create(config,
name=host_name)
# Persist the container info to the db with the ports
# Ports are used to determine what listeners to create
new_container = Container.create(
container_id=container.id,
name=host_name,
ip=ip,
mac=mac,
start_delay=image.start_delay,
start_retry_count=image.start_retry_count,
start_on_create=image.start_on_create,
sub_domain=image.sub_domain)
# Some containers perform a lot of startup work when run for the first time
# To mitigate this containers can be started and stopped on creation
if image.start_on_create:
await self.start_container_on_create(
image, new_container, client)
for port in image_ports[image.name]:
Port.create(container=new_container.id,
number=port.num,
protocol=port.protocol)
await client.close()
def create(self, the_parsed_topology_xml):
"""
Create a topology model here.
"""
# Instance a list of model.Component classes that are all the instanced items from parsed xml.
x = the_parsed_topology_xml
componentXMLNameToComponent = {
} #Dictionary mapss XML names to processes component objects so redundant processing is avoided
components = []
for comp_xml_path in x.get_comp_type_file_header_dict():
file_path = os.environ['BUILD_ROOT'] + '/' + comp_xml_path
print file_path
processedXML = XmlComponentParser.XmlComponentParser(file_path)
comp_name = processedXML.get_component().get_name()
componentXMLNameToComponent[comp_name] = processedXML
for instance in x.get_instances():
if instance.get_type() not in componentXMLNameToComponent.keys():
PRINT.info(
"Component XML file type {} was not specified in the topology XML. Please specify the path using <import_component_type> tags."
.format(instance.get_type()))
else:
instance.set_component_object(
componentXMLNameToComponent[instance.get_type()])
components.append(
Component.Component(instance.get_namespace(),
instance.get_name(),
instance.get_type(),
xml_filename=x.get_xml_filename(),
kind2=instance.get_kind()))
#print "Assembly name: " + x.get_name(), x.get_base_id(), x.get_base_id_window()
#for component in components:
# print component.get_name(), component.get_base_id(), component.get_base_id_window()
if self.__generate_new_IDS:
# Iterate over all the model.Component classes and...
instance_name_base_id_list = self.__compute_base_ids(
x.get_base_id(), x.get_base_id_window(), x.get_instances(),
x.get_xml_filename())
else:
instance_name_base_id_list = []
# Iterate over all the model.Component classes and then...
# Iterate over all the connection sources and assigne output ports to each Component..
# For each output port you want to assign the connect comment, target component namem, target port and type...
# (Requires adding to the model.Port class ether members or a memeber called of type TargetConnection)
for component in components:
port_obj_list = []
for connection in x.get_connections():
if component.get_name() == connection.get_source()[0]:
port = Port.Port(connection.get_source()[1],
connection.get_source()[2],
None,
None,
comment=connection.get_comment(),
xml_filename=x.get_xml_filename)
port.set_source_num(connection.get_source()[3])
port.set_target_comp(connection.get_target()[0])
port.set_target_port(connection.get_target()[1])
port.set_target_type(connection.get_target()[2])
port.set_target_num(connection.get_target()[3])
if connection.get_source()[1].startswith("i"):
port.set_direction("input")
else:
port.set_direction("output")
if connection.get_target()[1].startswith("i"):
port.set_target_direction("input")
else:
port.set_target_direction("output")
port_obj_list.append(port)
component.set_ports(port_obj_list)
# Instance a Topology class and give it namespace, comment and list of components.
the_topology = Topology.Topology(x.get_namespace(), x.get_comment(),
components, x.get_name(),
instance_name_base_id_list,
x.get_prepend_instance_name())
the_topology.set_instance_header_dict(
x.get_comp_type_file_header_dict())
return the_topology