def load_yaml_dump(filename):
'''
Load a YAML file and return a dict representation
'''
# Load YAML file
with open(filename, 'r') as infile:
return yaml.safe_load(infile)
def read_config(filename):
from pyaml import yaml
# Read the configuration file
with open(filename) as config_fd:
config = yaml.safe_load(config_fd)
return config
def set_seq_update(collection, seq_update):
"""
Set seq_update for current collection.
"""
with open(CONFIG_FILE, "r") as fl:
config = yaml.safe_load(fl)
config["collections"][collection]['seqUpdate'] = seq_update
with open(CONFIG_FILE, "w") as fl:
yaml.dump(config, fl, default_flow_style=False)
def read_nodes(nodes_filename):
"""
Convert a list of node names into a list of IP addresses.
:param nodes_filename: Name of the YAML file containing the
IP addresses
:type nodes_filename: str
:return: Tuple (List of IP addresses, Locator bits, uSID ID bits)
:rtype: tuple
:raises NodeNotFoundError: Node name not found in the mapping file
:raises InvalidConfigurationError: The mapping file is not a valid
YAML file
"""
# Read the mapping from the file
with open(nodes_filename, 'r') as nodes_file:
nodes = yaml.safe_load(nodes_file)
# Validate the IP addresses
for addr in [node['grpc_ip'] for node in nodes['nodes'].values()]:
if not utils.validate_ipv6_address(addr):
logger.error('Invalid IPv6 address %s in %s',
addr, nodes_filename)
raise InvalidConfigurationError
# Validate the SIDs
for sid in [node['uN'] for node in nodes['nodes'].values()]:
if not utils.validate_ipv6_address(sid):
logger.error('Invalid SID %s in %s',
sid, nodes_filename)
raise InvalidConfigurationError
# Validate the forwarding engine
for fwd_engine in [node['fwd_engine'] for node in nodes['nodes'].values()]:
if fwd_engine not in ['linux', 'vpp', 'p4']:
logger.error('Invalid forwarding engine %s in %s',
fwd_engine, nodes_filename)
raise InvalidConfigurationError
# Get the #bits of the locator
locator_bits = nodes.get('locator_bits')
# Validate #bits for the SID Locator
if locator_bits is not None and \
(int(locator_bits) < 0 or int(locator_bits) > 128):
raise InvalidConfigurationError
# Get the #bits of the uSID identifier
usid_id_bits = nodes.get('usid_id_bits')
# Validate #bits for the uSID ID
if usid_id_bits is not None and \
(int(usid_id_bits) < 0 or int(usid_id_bits) > 128):
raise InvalidConfigurationError
if locator_bits is not None and usid_id_bits is not None and \
int(usid_id_bits) + int(locator_bits) > 128:
raise InvalidConfigurationError
# Enforce case-sensitivity
for node in nodes['nodes'].values():
nodes['nodes'][node['name']]['grpc_ip'] = node['grpc_ip'].lower()
nodes['nodes'][node['name']]['uN'] = node['uN'].lower()
# Return the nodes list
return nodes['nodes'], locator_bits, usid_id_bits
def print_node_to_addr_mapping(nodes_filename):
"""
This function reads a YAML file containing the mapping
of node names to IP addresses and pretty print it
:param node_to_addr_filename: Name of the YAML file containing the
mapping of node names to IP addresses
:type node_to_addr_filename: str
"""
# Read the mapping from the file
with open(nodes_filename, 'r') as nodes_file:
nodes = yaml.safe_load(nodes_file)
# Validate the IP addresses
for addr in [node['grpc_ip'] for node in nodes['nodes'].values()]:
if not utils.validate_ipv6_address(addr):
logger.error('Invalid IPv6 address %s in %s',
addr, nodes_filename)
raise InvalidConfigurationError
# Validate the SIDs
for sid in [node['uN'] for node in nodes['nodes'].values()]:
if not utils.validate_ipv6_address(sid):
logger.error('Invalid SID %s in %s',
sid, nodes_filename)
raise InvalidConfigurationError
# Validate the forwarding engine
for fwd_engine in [node['fwd_engine'] for node in nodes['nodes'].values()]:
if fwd_engine not in ['linux', 'vpp', 'p4']:
logger.error('Invalid forwarding engine %s in %s',
fwd_engine, nodes_filename)
raise InvalidConfigurationError
# Get the #bits of the locator
locator_bits = nodes.get('locator_bits')
# Validate #bits for the SID Locator
if locator_bits is not None and \
(int(locator_bits) < 0 or int(locator_bits) > 128):
raise InvalidConfigurationError
# Get the #bits of the uSID identifier
usid_id_bits = nodes.get('usid_id_bits')
# Validate #bits for the uSID ID
if usid_id_bits is not None and \
(int(usid_id_bits) < 0 or int(usid_id_bits) > 128):
raise InvalidConfigurationError
if locator_bits is not None and usid_id_bits is not None and \
int(usid_id_bits) + int(locator_bits) > 128:
raise InvalidConfigurationError
print('\nList of available devices:')
pprint.PrettyPrinter(indent=4).pprint(list(nodes['nodes'].keys()))
print()
def add_hosts(nodes, edges, hosts_yaml):
'''
Add hosts to a topology
'''
# Read hosts information from a YAML file and
# add hosts to the nodes and edges lists
logger.info('*** Adding hosts to the topology')
# Open hosts file
with open(hosts_yaml, 'r') as infile:
hosts = yaml.safe_load(infile.read())
# Add hosts and links
for host in hosts:
# Add host
nodes.append({
'_key': host['name'],
'type': 'host',
'ip_address': host['ip_address']
})
# Get the subnet
net = str(ipaddress.ip_network(host['ip_address'], strict=False))
# Add edge (host to router)
# Character '/' is not accepted in key strign in arango, using
# '-' instead
edges.append({
'_key': '%s-dir1' % net.replace('/', '-'),
'_from': 'nodes/%s' % host['name'],
'_to': 'nodes/%s' % host['gw'],
'type': 'edge'
})
# Add edge (router to host)
# This is required because we work with
# unidirectional edges
edges.append({
'_key': '%s-dir2' % net.replace('/', '-'),
'_to': 'nodes/%s' % host['gw'],
'_from': 'nodes/%s' % host['name'],
'type': 'edge'
})
logger.info('*** Nodes YAML updated\n')
logger.info('*** Edges YAML updated\n')
# Return the updated nodes and edges lists
return nodes, edges
def fill_ip_addresses(nodes, addresses_yaml):
'''
Add addresses to a nodes dict
'''
# Read IP addresses information from a YAML file and
# add addresses to the nodes
logger.info('*** Filling nodes YAML file with IP addresses')
# Open hosts file
with open(addresses_yaml, 'r') as infile:
addresses = yaml.safe_load(infile.read())
# Parse addresses
node_to_addr = dict()
for addr in addresses:
node_to_addr[addr['node']] = addr['ip_address']
# Fill addresses
for node in nodes:
# Update the dict
node['ip_address'] = node_to_addr[node['_key']]
logger.info('*** Nodes YAML updated\n')
# Return the updated nodes list
return nodes
def get_seq_update(collection):
"""
Get seqUpdate from file if no -> get from server with default date -> if no def date get (current date - 3)
"""
with open(CONFIG_FILE, "r") as configuration_file:
configuration = yaml.safe_load(configuration_file)
seq_update = configuration['collections'][collection]['seqUpdate']
if seq_update is not None:
return seq_update
else:
default_date = configuration['collections'][collection]['default_date']
if default_date is None:
default_date = (datetime.datetime.now() - datetime.timedelta(days=3)).strftime("%Y-%m-%d")
configuration['collections'][collection]['default_date'] = default_date
with open(CONFIG_FILE, "w") as configuration_file:
yaml.dump(configuration, configuration_file, default_flow_style=False)
seq_update = poller.get_seq_update_by_date(collection, default_date)
set_seq_update(collection, seq_update)
return seq_update
else:
seq_update = poller.get_seq_update_by_date(collection, default_date)
set_seq_update(collection, seq_update)
return seq_update
def get_parsed_student(cls, file_path: str) -> Dict[str, str]:
with open(file_path, "r") as f:
return yaml.safe_load(f)
def set_wells():
with open("input/wells.yml", 'r') as stream:
wells_loaded = yaml.safe_load(stream)
"malware/malware": ["name", "platform", "shortDescription", "threatLevel"],
"malware/targeted_malware": ["date", "malware/name", "md5", "injectMd5", "threatActor/name", "fileName", "fileType", "size", "source" ],
"suspicious_ip/tor_node": ["dateFirstSeen", "dateLastSeen", "ipv4/ip", "source"],
"suspicious_ip/open_proxy": ["dateDetected", "dateFirstSeen", "ipv4/ip", "type", "port", "source"],
"suspicious_ip/socks_proxy": ["dateDetected", "dateFirstSeen", "ipv4/ip", "source"]
}
additional_fields = {
"apt/threat/mitre_matrix": ["threat/id", "threatActor/name", "title", "attackTactic", "attackType", "tacticId"],
"hi/threat/mitre_matrix": ["threat/id", "threatActor/name", "title", "attackTactic", "attackType", "tacticId"]
}
with open("configuration.yml", "r") as configuration_file:
config = yaml.safe_load(configuration_file)
API_URL = config["client"]["api_url"] if config["client"]["api_url"][-1] == "/" else config["client"]["api_url"] + "/"
API_USERNAME = config["client"]["username"]
API_KEY = config["client"]["api_key"]
BIG_DATA_COLLECTIONS = ["hi/threat", "apt/threat"]
LIMIT = config["client"]["default_limit"]
BIG_DATA_LIMIT = config["client"]["big_data_limit"]
CONFIG_FILE = "configuration.yml"
DATA_DIR = config["client"]["data_dir"]
PROXY_PROTOCOL = config["proxy"]["protocol"]
PROXY_ADDRESS = config["proxy"]["ip_addr"]
PROXY_PORT = config["proxy"]["port"]
PROXY_USERNAME = config["proxy"]["username"]
PROXY_PASSWORD = config["proxy"]["password"]
from pyaml import yaml
emp_dict = {'name': 'Theja', 'age': 35, 'salary': 1000.0, 'isMarried': True}
yaml_string = yaml.dump(emp_dict)
print(yaml_string)
with open('emp.yaml', 'w') as f:
yaml.dump(emp_dict, f)
# ed = yaml.load(yaml_string)
# /Code/venv/serialization/10Yaml.py:<>: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.
# ed = yaml.load(yaml_string)
ed = yaml.safe_load(yaml_string)
# print('type(ed)', type(ed))
print(ed)
for k, v in ed.items():
print(k, ':', v)
with open('emp.yaml', 'r') as f:
edf = yaml.safe_load(f)
print(edf)
# age: 35
# isMarried: true
# name: Theja
# salary: 1000.0
#
# type(ed) <class 'dict'>
from pyaml import yaml
with open("student.yaml", "r") as f:
yaml_string = yaml.safe_load(f)
print(yaml_string)
from pyaml import yaml
STUDENT = {
'name': 'Mark',
'age': 22,
'spec': 'math',
'fee': 1000.0,
'isPass': True,
'backlogs': None
}
yaml_string = yaml.dump(STUDENT)
print(yaml_string)
new_dect = yaml.safe_load(yaml_string)
print(type(new_dect))
print(new_dect)
for key, val in new_dect.items():
print(key, '....', val)
from pyaml import yaml
emp_dict = {'name': 'Surender', 'age': 20, 'salary': 1000.0, 'IsMarried': True}
#serialization to yaml string
yaml_string = yaml.dump(emp_dict)
print(yaml_string)
# #serilaization to a yaml lfile
with open('emp.yaml', 'w') as f:
yaml.dump(emp_dict, f)
#Deserialization from yamaml string
new_dict = yaml.safe_load(yaml_string)
print(new_dict)
for k, v in new_dict.items():
print(k, ':', v)
#Deserialization from yamaml file
with open('emp.yaml', 'r') as f:
new_dict2 = yaml.safe_load(f)
print(new_dict2)
def p5_reference():
with PYP5JS_FILES.p5_yml.open('r') as fd:
return yaml.safe_load(fd)
