def main():
if len(sys.argv) < 2:
usage()
sys.exit(-1)
commands = []
print(f"Loading {sys.argv[1]}")
service = yaml_load(sys.argv[1])
language = service["language"]
print(f"Merging {language}.yaml")
base_yaml = yaml_load(f'{package_data(language)}.yaml')
service = yaml_load(sys.argv[1], base_yaml)
print("Result:")
pprint.PrettyPrinter(indent=2).pprint(service)
components = service["components"]
commands.append(base_image(language))
commands.append(install_system_components(components))
commands.extend(create_unprivileged_user())
if is_jvm(language):
agents = service["agents"]
commands.extend(jvm_agents(agents))
commands.extend(add_entrypoint(language))
dockerfile_api.write("Dockerfile", commands)
print("Dockerfile written!")
def get_all_configs():
full_config = {}
for config_path in get_all_config_locations():
if not full_config:
full_config = yaml_load(config_path)
else:
full_config = yaml_load(config_path, full_config)
return full_config
def test_falsy_default_config_works(self):
default_config = {}
config = yaml_load("testdata/empty_yaml_dir", default_config)
self.assertEqual(config, default_config)
default_config = None
config = yaml_load("testdata/empty_yaml_dir", default_config)
self.assertEqual(config, default_config)
def _check_duplicate_account_names(yaml_path, mock_data_merge):
yamlreader.yaml_load(yaml_path)
account_names_found = set()
for args, kwargs in mock_data_merge.call_args_list:
_, new_data = args
names = new_data.keys()
for name in names:
if name in account_names_found:
raise Exception("Duplicate definition of account %r" % name)
account_names_found.add(name)
def load_config(global_config_dir=CFGDIR):
global_config = {}
if os.path.isdir(global_config_dir):
global_config = yamlreader.yaml_load(global_config_dir, {})
user_config = {}
user_config_dir = os.path.expanduser("~/.afp-cli")
if os.path.isdir(user_config_dir):
global_config = yamlreader.yaml_load(user_config_dir, {})
yamlreader.data_merge(global_config, user_config)
return global_config
def load_config(global_config_dir=CFGDIR):
global_config = {}
if os.path.isdir(global_config_dir):
global_config = yamlreader.yaml_load(global_config_dir, {})
user_config = {}
user_config_dir = os.path.expanduser("~/.afp-cli")
if os.path.isdir(user_config_dir):
user_config = yamlreader.yaml_load(user_config_dir, {})
yamlreader.data_merge(global_config, user_config)
return global_config
def process_nat11(dict_intent):
# loading YAML file with firewall settings
if dict_intent['protocol'] == 'all':
dict_intent['protocol'] = 'any'
config = yaml_load('cisco_config.yml')
# identifies interfaces
for interface in config['INTERFACES']:
if check_ip_network(dict_intent['from'], interface['addr']):
dict_intent['from_interface'] = interface['name']
elif check_ip_network(dict_intent['to'], interface['addr']):
dict_intent['to_interface'] = interface['name']
else:
return 'CISCO TRANSLATOR: IP/Network not recognized'
# loading and render template jinja2
file_loader = FileSystemLoader('.')
env = Environment(loader=file_loader)
template = env.get_template('cisco_template.j2')
output = template.render(dict_intent)
with ClusterRpcProxy(CONFIG) as rpc_connect:
rpc_connect.cisco_connector.apply_config(config['ip_manage'],
config['ssh_port'],
config['username'],
config['password'],
config['device_type'], output)
return output
def load_config(config_dir):
try:
return yamlreader.yaml_load(config_dir)
except Exception:
print("Could not load configuration from '{0}'".format(
config_dir), file=sys.stderr)
raise
def load_mutations(filepath):
# load mutations from config file
try:
config = yaml_load(filepath)
mutations = config.get('mutations', {})
except YamlReaderError:
logging.error("Error loading mutations from file {}.".format(filepath))
sys.exit(1)
return mutations
def read_config(config_path, defaults):
"""Read config file from given location, and parse properties"""
if not os.path.isdir(config_path):
raise ValueError("{0} is not a directory".format(config_path))
try:
return yaml_load(config_path, defaults)
except yaml.YAMLError:
logging.exception("Failed to read YAML config from directory: {0}".format(config_path))
def yml2csv(self):
config = yaml_load("../chatbotdata")
data = config["conversations"]
finalData = []
for item in data:
finaljson = {}
finaljson['Terms'] = item[0]
finaljson['Definitons'] = item[1]
finalData.append(finaljson)
def get_fbChatBot_yml_format(self):
config = yaml_load("../chatbot_corpus_data")
data = config["conversations"]
finalData = []
for item in data:
print(item)
finaljson = {}
finaljson[item[0]] = item[1]
finalData.append(finaljson)
return finalData
def read_directory(yaml_path):
"""Read yaml files and return merged yaml data"""
accounts = yamlreader.yaml_load(yaml_path)
for account in accounts.values():
account['id'] = str(account['id'])
_check_account_data(accounts)
logging.debug("Read yaml files from directory '%s'", yaml_path)
return accounts
def get(paths: list) -> dict:
for path in paths:
if not os.path.isfile(path):
raise ValueError(
f"The path {path} did not refer to any existing file")
config: dict = yamlreader.yaml_load(paths)
config["display"]["colours"]["background"] = tuple(
config["display"]["colours"]["background"])
return config
def read_directory(yaml_path):
"""Read yaml files and return merged yaml data"""
accounts = yamlreader.yaml_load(yaml_path)
for account in accounts.values():
account['id'] = str(account['id'])
_check_account_data(accounts)
_check_duplicate_account_names(yaml_path)
logging.debug("Read yaml files from directory '%s'", yaml_path)
return accounts
def initialize_federation_proxy(user=None):
"""Get needed config parts and initialize AWSFederationProxy"""
config_path = request.environ.get('CONFIG_PATH')
if config_path is None:
raise Exception("No Config Path specified")
config = yaml_load(config_path)
try:
logger = setup_logging(config, logger_name=LOGGER_NAME)
except Exception as exc:
raise ConfigurationError(str(exc))
if user is None:
user = get_user(config['api']['user_identification'])
account_config_path = request.environ.get('ACCOUNT_CONFIG_PATH')
if account_config_path is None:
raise Exception("No Account Config Path specified")
account_config = yaml_load(account_config_path)
proxy = AWSFederationProxy(user=user, config=config,
account_config=account_config, logger=logger)
return proxy
def process_nat11(dict_intent):
# loading YAML file with firewall settings
config = yaml_load('openflow_config.yml')
dict_intent['hostname'] = config['hostname']
# loading and render template jinja2
file_loader = FileSystemLoader('.')
env = Environment(loader=file_loader)
template = env.get_template('openflow_template.j2')
output = template.render(dict_intent)
#with ClusterRpcProxy(CONFIG) as rpc_connect:
# rpc_connect.linux_connector.apply_config(config['ip_manage'], config['ssh_port'], config['username'], config['password'],
# config['device_type'], output, 'openflow')
return output
def process_dst_route(dict_intent):
# loading YAML file with firewall settings
config = yaml_load('cisco_config.yml')
if 'to_mask' in dict_intent:
dict_intent['to'] = dict_intent['to'] + ' ' + dict_intent['to_mask']
# loading and render template jinja2
file_loader = FileSystemLoader('.')
env = Environment(loader=file_loader)
template = env.get_template('cisco_template.j2')
output = template.render(dict_intent)
#with ClusterRpcProxy(CONFIG) as rpc_connect:
# rpc_connect.cisco_connector.apply_config(config['ip_manage'], config['ssh_port'], config['username'],
# config['password'], config['device_type'], output)
return output
def initialize_federation_proxy(user=None):
"""Get needed config parts and initialize AWSFederationProxy"""
config_path = request.environ.get('CONFIG_PATH')
if config_path is None:
raise Exception("No Config Path specified")
config = yaml_load(config_path)
try:
logger = setup_logging(config, logger_name=LOGGER_NAME)
except Exception as exc:
raise ConfigurationError(str(exc))
if user is None:
user = get_user(config['api']['user_identification'])
account_config_path = request.environ.get('ACCOUNT_CONFIG_PATH')
if account_config_path is None:
raise Exception("No Account Config Path specified")
account_config = yaml_load(account_config_path)
proxy = AWSFederationProxy(user=user,
config=config,
account_config=account_config,
logger=logger)
return proxy
def load_manifests(path, glob='*.certs.yaml'):
"""
Given a directory, this will load all files matching config_file_glob
as YAML and then recursively merge them into a single config hash
using the yamlreader library.
:param glob default: '*.certs.yaml'
:return config dict
"""
path = os.path.abspath(path)
# If this is a single file, then no need to use the glob.
if os.path.isfile(path):
log.info(
'Loading certificate and authority manifest from {}'.format(path))
return yaml_load(path, default_config)
# Else it is a directory, probably containing multiple manifest files.
# Load any file that matches path glob.
else:
path_glob = os.path.join(path, glob)
log.info(
'Loading all certificate and authority manifests in {}'.format(
path_glob))
return yaml_load(path_glob, default_config)
def update_from_files():
""" merge config files from all available env vars """
dct = dict()
for env_var in ('APP_SETTINGS_YAML', 'CONFIG_YAML', 'SECRETS_YAML'):
if env_var in os.environ and os.path.exists(os.environ[env_var]):
dct = yaml_load(os.environ[env_var], dct)
# if start_at_operation_time is set in a config file, yamlreader converts
# an iso8601 datetime string into a `datetime` opbject
# We need to convert it to a `bson.timestamp.Timestamp`
options = dct.get('stream', {}).get('options', {})
date = options.get('start_at_operation_time')
if date:
dct['stream']['options']['start_at_operation_time'] = \
utc_timestamp(date)
update_from_dict(dct)
def load(self, filepath):
try:
config = yaml_load(filepath)
except IOError:
logging.error("Error opening config file {}.".format(filepath))
sys.exit(1)
try:
self.address = config["server"]["binding-address"]
self.port = config["server"]["port"]
self.cert = config["server"]["cert"]
self.key = config["server"]["key"]
self.mutations = config["server"]["mutations"]
except KeyError as err:
logging.error("Error loading configuration: {}".format(str(err)))
sys.exit(1)
def run(self):
handler = WatchedFileHandler('/tmp/cake.log')
self.logger = logging.getLogger()
self.logger.addHandler(handler)
try:
config = yamlreader.yaml_load('/etc/cake.conf.d/')
self.logger.exception(config)
while True:
for test_name, test_id in config['tests'].iteritems():
self.logger.info('{0}: fetching data for {1}:{2}'.format(
time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()), test_name, test_id))
data_dict = gather_data(config['apikey'], config['username'], test_id)
push_data(data_dict, test_name, config['graphitehost'])
time.sleep(600)
except Exception:
self.logger.exception('succubus doing some strange stuff')
def get_config_from_file(self, filename):
"""Return the host configuration from the hostfile.
Parses the given YAML-File and returns the content. Will log a warning in case of malformed
YAML or missing file and return and empty dict
Args:
filename (str): a filname (with path) to Read
Returns:
on success: a dict of attributes
on failure: an empty dict
"""
try:
return yaml_load(filename) or {}
except YamlReaderError as exc:
self.warning("Hostfile '{}' not a proper YAML-File: {}".format(
filename, exc))
return {}
def read_config(config_path):
"""Read config file from given location, and parse properties"""
if not os.path.isdir(config_path):
raise ValueError('{0} is not a directory'.format(config_path))
defaultconfig = {
"mqtt": {
"host": "localhost",
"base_topic": "bbq"
},
"scale": "fahrenheit",
"interval": 15
}
try:
return yaml_load(config_path, defaultconfig)
except yaml.YAMLError:
logging.exception('Failed to parse configuration directory:')
def process_traffic_shaping(dict_intent):
# loading YAML file with firewall settings
config = yaml_load('cisco_config.yml')
# converting throughput and rate
dict_intent['with'] = dict_intent['with'] * 1000000
dict_intent['rate'] = int(dict_intent['with'] * 0.0005)
# range/host treatment
if 'from_mask' in dict_intent:
dict_intent[
'from'] = dict_intent['from'] + ' ' + dict_intent['from_mask']
else:
dict_intent['from'] = 'host ' + dict_intent['from']
if 'to_mask' in dict_intent:
dict_intent['to'] = dict_intent['to'] + ' ' + dict_intent['to_mask']
else:
dict_intent['to'] = 'host ' + dict_intent['to']
# define order
order = define_order(dict_intent)
if order != 0:
dict_intent['order'] = order
else:
return 'CISCO TRANSLATOR - ERROR ORDER: It was not possible to determine the order by name in order parameter'
# translate protocol/port
if dict_intent['traffic'] == 'all':
dict_intent['traffic'] = 'ip'
elif dict_intent['traffic'] == 'icmp':
dict_intent['traffic'] = 'icmp'
else:
protocol, port = dict_intent['traffic'].split('/')
dict_intent['traffic'] = protocol
dict_intent['traffic_port'] = 'eq ' + port
# loading and render template jinja2
file_loader = FileSystemLoader('.')
env = Environment(loader=file_loader)
template = env.get_template('cisco_template.j2')
output = template.render(dict_intent)
with ClusterRpcProxy(CONFIG) as rpc_connect:
rpc_connect.cisco_connector.apply_config(config['ip_manage'],
config['ssh_port'],
config['username'],
config['password'],
config['device_type'], output)
return output
def process_nat11(dict_intent):
config = yaml_load('iptables_config.yml')
for interface in config['INTERFACES']:
if check_ip_network(dict_intent['from'], interface['addr']):
dict_intent['from_interface'] = interface['name']
elif check_ip_network(dict_intent['to'], interface['addr']):
dict_intent['to_interface'] = interface['name']
else:
return 'CISCO TRANSLATOR: IP/Network not recognized'
# loading and render template jinja2
file_loader = FileSystemLoader('.')
env = Environment(loader=file_loader)
template = env.get_template('iptables_template.j2')
output = template.render(dict_intent)
with ClusterRpcProxy(CONFIG) as rpc_connect:
rpc_connect.linux_connector.apply_config(
config['ip_manage'], config['ssh_port'], config['username'],
config['password'], config['device_type'], output, 'iptables')
return output
def process_url_filter(dict_intent):
# loading YAML file with firewall settings
config = yaml_load('paloalto_config.yml')
if dict_intent['from'] == 'all':
dict_intent['from'] = 'any'
# identifies the use of ranges
if 'from_mask' in dict_intent:
dict_intent[
'from'] = dict_intent['from'] + ' ' + dict_intent['from_mask']
# other configs
dict_intent['password'] = config['password']
# loading and render template jinja2
file_loader = FileSystemLoader('.')
env = Environment(loader=file_loader)
template = env.get_template('paloalto_template.j2')
output = template.render(dict_intent)
#with ClusterRpcProxy(CONFIG) as rpc_connect:
# rpc_connect.cisco_connector.apply_config(config['ip_manage'], config['ssh_port'], config['username'], config['password'], config['device_type'], output)
return output
def run(self):
handler = WatchedFileHandler('/tmp/cake.log')
self.logger = logging.getLogger()
self.logger.addHandler(handler)
try:
config = yamlreader.yaml_load('/etc/cake.conf.d/')
self.logger.exception(config)
while True:
for test_name, test_id in config['tests'].iteritems():
self.logger.info('{0}: fetching data for {1}:{2}'.format(
time.strftime("%a, %d %b %Y %H:%M:%S +0000",
time.gmtime()), test_name, test_id))
data_dict = gather_data(config['apikey'],
config['username'], test_id)
push_data(data_dict, test_name, config['graphitehost'])
time.sleep(600)
except Exception:
self.logger.exception('succubus doing some strange stuff')
def main(env_name, mode, path, iteration, job_name, horizon, run_no):
env_kwargs = {}
if path and ('.pickle' in path or 'pkl' in path):
policy_path = path
else:
if job_name:
path = os.path.join('../inverse_rl_dexterous_hand/training/Runs/',
job_name, 'run_' + str(run_no), 'iterations')
if iteration:
if iteration == 'last':
checkpoint_file = get_last_iteration_checkpoint(path)
else:
checkpoint_file = "checkpoint_{}.pickle".format(iteration)
policy_path = os.path.join(path, checkpoint_file)
else:
policy_path = os.path.join(path, "best_policy.pickle")
if env_name is None:
cfg_path = os.path.join(os.path.dirname(policy_path), "../..", "..",
"config.yaml")
if not os.path.exists(cfg_path):
cfg_path = os.path.join(os.path.dirname(cfg_path), "../..",
"config.yaml")
if not os.path.exists(cfg_path):
cfg_path = None
if cfg_path is not None:
cfg = yamlreader.yaml_load(cfg_path)
env_name = cfg['env']
env_kwargs = cfg['env_kwargs']
else:
print(
"Config file not found, cannot infer environment name. Please provide env_name parameter."
)
exit(1)
e = GymEnv(env_name, **env_kwargs)
print("Checkpoint path:", policy_path)
policy = pickle.load(open(policy_path, 'rb'))
if isinstance(policy, list):
policy = policy[0]
# render policy
if horizon is None:
horizon = e.horizon
e.visualize_policy(policy, num_episodes=100, horizon=horizon, mode=mode)
def _read_config(config_path):
"""Read config file from given location, and parse properties"""
if config_path is not None:
if os.path.isfile(config_path):
logging.info(f'Config file found at: {config_path}')
try:
with open(config_path, 'r') as f:
return yaml.safe_load(f.read())
except yaml.YAMLError:
logging.exception('Failed to parse configuration file:')
elif os.path.isdir(config_path):
logging.info(f'Config directory found at: {config_path}')
try:
return yaml_load(config_path)
except yaml.YAMLError:
logging.exception('Failed to parse configuration directory:')
return {}
def __main():
config_data = vars(__parse_args())
config_data = {key: config_data[key] for key in config_data
if config_data[key] is not None}
if 'config' in config_data:
try:
config_data = data_merge(yaml_load(config_data['config']),
config_data)
except YamlReaderError as exc:
raise("Could not read configfile: %s" % exc)
if len(config_data['input']) == 0:
raise("No input direcotires configured.")
if 'gendersfile' not in config_data:
config_data['gendersfile'] = '/etc/genders'
genders_generator = GenerateGenders(
config_data.get('input'),
config_data.get('gendersfile'),
config_data.get('domain', {}),
config_data.get('verbosity')
)
genders_generator.generate_genders_file()
def process_dst_route(dict_intent):
# loading YAML file with firewall settings
config = yaml_load('iptables_config.yml')
# loading and render template jinja2
for interface in config['INTERFACES']:
if check_ip_network(dict_intent['from'], interface['addr']):
dict_intent['interface'] = interface['name']
if 'interface' not in dict_intent:
return "IPTABLES TRANSLATOR: Unrecognized gateway"
dict_intent['to'] = dict_intent['to'] + '/' + str(
IPAddress(dict_intent['to_mask']).netmask_bits())
file_loader = FileSystemLoader('.')
env = Environment(loader=file_loader)
template = env.get_template('iptables_template.j2')
output = template.render(dict_intent)
with ClusterRpcProxy(CONFIG) as rpc_connect:
rpc_connect.linux_connector.apply_config(config['ip_manage'],
config['ssh_port'],
config['username'],
config['password'],
config['device_type'], output)
return output
#!/usr/bin/python3
import sys
import yaml
import glob
import json
from yamlreader import yaml_load
from datetime import datetime, date
today = date.today()
events = yaml_load("events/*.yaml")
output = open("events.json", "w")
okevents = {}
def eventdate(elem):
return elem['StartDate']
# Ugly, we loop over events multiple times,
# but I couldn't get it working in 1 pass...
for event in events:
if event['EndDate'] < event['StartDate']:
print("Event ends before it starts\n")
events.remove(event)
for event in events:
if today > event['EndDate']:
def load_config(self, config_path):
basic_loaded_config = yamlreader.yaml_load(config_path)
return dict(((k.replace('-', '_'), v)
for k, v in basic_loaded_config.items()))
def get_config():
config_path = request.environ.get('CONFIG_PATH', '/etc/afp-resource-maker')
return yaml_load(config_path)
def process_acl(dict_intent):
# loading YAML with openflow settings
config = yaml_load('openflow_config.yml')
dict_intent['hostname'] = config['hostname']
dict_intent['args'] = ''
# from/to
if 'from_mask' in dict_intent:
dict_intent[
'from'] = dict_intent['from'] + '/' + dict_intent['from_mask']
if 'to_mask' in dict_intent:
dict_intent['to'] = dict_intent['to'] + '/' + dict_intent['to_mask']
if dict_intent['from'] != 'all':
dict_intent['args'] = dict_intent['args'] + 'nw_src=' + dict_intent[
'from'] + ','
else:
dict_intent['args'] = dict_intent['args'] + 'nw_src=0.0.0.0/0.0.0.0,'
if dict_intent['to'] != 'all':
dict_intent[
'args'] = dict_intent['args'] + 'nw_dst=' + dict_intent['to'] + ','
else:
dict_intent['args'] = dict_intent['args'] + 'nw_dst=0.0.0.0/0.0.0.0,'
# translate protocol/port
if dict_intent['traffic'] == 'icmp':
dict_intent['args'] = dict_intent['args'] + 'nw_proto=1,icmp_type=8,'
elif dict_intent['traffic'] != 'all':
protocol, port = dict_intent['traffic'].split('/')
dict_intent['traffic'] = protocol
dict_intent['traffic_port'] = 'eq ' + port
if protocol == 'tcp':
dict_intent['args'] = dict_intent[
'args'] + 'nw_proto=6,tcp_dst=' + port + ','
elif protocol == 'udp':
dict_intent['args'] = dict_intent[
'args'] + 'nw_proto=17,udp_dst=' + port + ','
if dict_intent['rule'] == 'block':
dict_intent['rule'] = 'drop'
else:
dict_intent['rule'] = 'normal'
order = define_order(dict_intent)
if order != 0:
dict_intent['order'] = order
else:
return 'OPENFLOW TRANSLATOR - ERROR ORDER: It was not possible to determine the order by name in order parameter'
file_loader = FileSystemLoader('.')
env = Environment(loader=file_loader)
template = env.get_template('openflow_template.j2')
response = template.render(dict_intent)
output = 'ovs-ofctl del-flows ' + dict_intent['hostname'] + '\n'
output = output + response + '\n'
lines = ['# log acl rules \n']
file = 'rules/openflow_acls'
with open(file) as archive:
for line in archive:
if '#' not in line[0:5] and line[0:1] != "\n":
dict_rule = ast.literal_eval(line)
if int(dict_rule['order']) > int(dict_intent['order']):
tmp = template.render(dict_rule)
output = output + tmp + '\n'
lines.append(str(dict_rule) + '\n')
elif int(dict_rule['order']) <= int(dict_intent['order']):
if dict_rule['name'] == dict_intent['name']:
lines.append(str(dict_rule) + '\n')
else:
if dict_intent['apply'] == 'insert':
dict_rule['order'] = int(dict_rule['order']) - 1
else:
dict_rule['order'] = int(dict_rule['order']) + 1
tmp = template.render(dict_rule)
output = output + tmp + '\n'
lines.append(str(dict_rule) + '\n')
archive.close()
archive = open(file, 'w')
archive.writelines(lines)
archive.close()
output = output + '\novs-ofctl add-flow ' + dict_intent[
'hostname'] + ' priority=0,action=normal'
#with ClusterRpcProxy(CONFIG) as rpc_connect:
# rpc_connect.linux_connector.apply_config(config['ip_manage'], config['ssh_port'], config['username'], config['password'],
# config['device_type'], output, 'openflow')
return response