def test_release(self, mock_get_version):
vpn = VPN(host='localhost', port=1234)
self.assertIsNone(vpn._release)
release_string = 'OpenVPN 2.4.4 x86_64-pc-linux-gnu [SSL (OpenSSL)] [LZO] [LZ4] [EPOLL] [PKCS11] [MH/PKTINFO] [AEAD] built on Sep 5 2018'
mock_get_version.return_value = release_string
self.assertEqual(vpn.release, release_string)
self.assertEqual(vpn._release, release_string)
mock_get_version.assert_called_once_with()
mock_get_version.reset_mock()
vpn._release = 'asd'
self.assertEqual(vpn.release, 'asd')
mock_get_version.assert_not_called()
def load_defaults(self):
"""Load default config options.
"""
# Settings
self.settings = {}
self.settings['name'] = None
self.settings['geoip_data'] = None
self.settings['datetime_format'] = '%d/%m/%Y %H:%M:%S'
# VPN(s)
self.vpns = []
vpn = VPN(host='localhost', port=5555)
vpn.name = 'Default VPN'
vpn.allow_disconnect = True
self.vpns.append(vpn)
def test_socket(self):
vpn = VPN(socket='file.sock')
self.assertEqual(vpn._mgmt_socket, 'file.sock')
self.assertIsNone(vpn._mgmt_host)
self.assertIsNone(vpn._mgmt_port)
self.assertEqual(vpn.type, VPNType.UNIX_SOCKET)
self.assertEqual(vpn.mgmt_address, 'file.sock')
def test_host_port(self):
vpn = VPN(host='localhost', port=1234)
self.assertEqual(vpn._mgmt_host, 'localhost')
self.assertEqual(vpn._mgmt_port, 1234)
self.assertIsNone(vpn._mgmt_socket)
self.assertEqual(vpn.type, VPNType.IP)
self.assertEqual(vpn.mgmt_address, 'localhost:1234')
def test__get_version(self, mock_send_command):
vpn = VPN(host='localhost', port=1234)
mock_send_command.return_value = """
OpenVPN Version: OpenVPN 2.4.4 x86_64-pc-linux-gnu [SSL (OpenSSL)] [LZO] [LZ4] [EPOLL] [PKCS11] [MH/PKTINFO] [AEAD] built on Sep 5 2018
Management Version: 1
END
"""
self.assertEqual(
vpn._get_version(),
'OpenVPN 2.4.4 x86_64-pc-linux-gnu [SSL (OpenSSL)] [LZO] [LZ4] [EPOLL] [PKCS11] [MH/PKTINFO] [AEAD] built on Sep 5 2018'
)
mock_send_command.assert_called_once_with('version')
mock_send_command.reset_mock()
mock_send_command.return_value = ""
with self.assertRaises(ParseError) as ctx:
vpn._get_version()
self.assertEqual(
'Unable to get OpenVPN version, no matches found in socket response.',
str(ctx.exception))
mock_send_command.assert_called_once_with('version')
mock_send_command.reset_mock()
mock_send_command.return_value = """
Management Version: 1
END
"""
with self.assertRaises(ParseError) as ctx:
vpn._get_version()
self.assertEqual(
'Unable to get OpenVPN version, no matches found in socket response.',
str(ctx.exception))
mock_send_command.assert_called_once_with('version')
mock_send_command.reset_mock()
def test_cache(self, release_mock, state_mock):
"""Test caching VPN metadata works and clears correctly.
"""
vpn = VPN(host='localhost', port=1234)
vpn.cache_data()
release_mock.assert_called_once()
state_mock.assert_called_once()
vpn._release = 'asd'
vpn._state = 'qwe'
vpn.clear_cache()
self.assertIsNone(vpn._release)
self.assertIsNone(vpn._state)
def do_config(self):
# check status
start_time = time.time()
LOG.info("check vpn status, vpn: %s" % self.host_ip)
commonutils.check_host_status(self.host_ip,
self.user,
self.password,
retry_time=500,
interval=1)
LOG.info("vpn is ready, vpn: %s" % self.host_ip)
LOG.info("start config vpn, vpn: %s, add_conns: %s" %
(self.host_ip, self.add_conns))
# add_conn
vpn = VPN(public_ip=self.host_ip,
user=self.user,
pass_word=self.password)
for conn in self.add_conns:
try:
vpn.add_tunnel(tunnel_name=conn["tunnel_name"],
left=conn["left_public_ip"],
left_subnet=conn["left_subnet"],
right=conn["right_public_ip"],
right_subnet=conn["right_subnet"])
except Exception as e:
LOG.error("add conn error, vpn: %s, conn: %s, error: %s" %
(self.host_ip, conn, e.message))
LOG.info("add conns success, vpn: %s, conns: %s" %
(self.host_ip, self.add_conns))
# delete conns
for conn in self.delete_conns:
try:
vpn.remove_tunnel(conn)
except Exception as e:
LOG.error("delete conn error, vpn: %s, conn: %s, error: %s" %
(self.host_ip, conn, e.message))
LOG.info("delete conns success, vpn: %s, conns: %s" %
(self.host_ip, self.delete_conns))
try:
vpn.restart_ipsec_service()
except Exception as e:
LOG.error("restart ipsec error, vpn: %s, error: %s" %
(self.host_ip, e.message))
cost_time = time.time() - start_time
LOG.info("config vpn success, vpn: %s, cost time: %d" %
(self.host_ip, cost_time))
def test_version(self, mock_get_version):
vpn = VPN(host='localhost', port=1234)
vpn._release = 'OpenVPN 2.4.4 x86_64-pc-linux-gnu [SSL (OpenSSL)] [LZO] [LZ4] [EPOLL] [PKCS11] [MH/PKTINFO] [AEAD] built on Sep 5 2018'
self.assertEqual(vpn.version, '2.4.4')
vpn._release = 'OpenVPN 1.2.3 x86_64-pc-linux-gnu [SSL (OpenSSL)] [LZO] [LZ4] [EPOLL] [PKCS11] [MH/PKTINFO] [AEAD] built on Sep 5 2018'
self.assertEqual(vpn.version, '1.2.3')
vpn._release = 'OpenVPN 11.22.33 x86_64-pc-linux-gnu [SSL (OpenSSL)] [LZO] [LZ4] [EPOLL] [PKCS11] [MH/PKTINFO] [AEAD] built on Sep 5 2018'
self.assertEqual(vpn.version, '11.22.33')
vpn._release = None
mock_get_version.assert_not_called(
) # Check mock hasn't been triggered up to this point
mock_get_version.return_value = None
self.assertIsNone(vpn.version)
mock_get_version.assert_called_once()
mock_get_version.reset_mock()
vpn._release = 'asd'
with self.assertRaises(ParseError) as ctx:
vpn.version()
self.assertEqual('Unable to parse version from release string.',
str(ctx.exception))
mock_get_version.assert_not_called()
def do_config(self):
# check status
start_time = time.time()
LOG.info("check vpn status, vpn: %s" % self.host_ip)
commonutils.check_host_status(self.host_ip, self.user, self.password,
retry_time=500, interval=1)
LOG.info("vpn is ready, vpn: %s" % self.host_ip)
LOG.info("start config vpn, vpn: %s, add_conns: %s"
% (self.host_ip, self.add_conns))
# add_conn
vpn = VPN(public_ip=self.host_ip, user=self.user,
pass_word=self.password)
for conn in self.add_conns:
try:
vpn.add_tunnel(tunnel_name=conn["tunnel_name"],
left=conn["left_public_ip"],
left_subnet=conn["left_subnet"],
right=conn["right_public_ip"],
right_subnet=conn["right_subnet"])
except Exception as e:
LOG.error("add conn error, vpn: %s, conn: %s, error: %s"
% (self.host_ip, conn, e.message))
LOG.info("add conns success, vpn: %s, conns: %s"
% (self.host_ip, self.add_conns))
# delete conns
for conn in self.delete_conns:
try:
vpn.remove_tunnel(conn)
except Exception as e:
LOG.error("delete conn error, vpn: %s, conn: %s, error: %s"
% (self.host_ip, conn, e.message))
LOG.info("delete conns success, vpn: %s, conns: %s"
% (self.host_ip, self.delete_conns))
try:
vpn.restart_ipsec_service()
except Exception as e:
LOG.error("restart ipsec error, vpn: %s, error: %s"
% (self.host_ip, e.message))
cost_time = time.time() - start_time
LOG.info("config vpn success, vpn: %s, cost time: %d"
% (self.host_ip, cost_time))
def test_server_state(self, mock):
vpn = VPN(host='localhost', port=1234)
mock.return_value = """1560719601,CONNECTED,SUCCESS,10.0.0.1,,,1.2.3.4,1194
END"""
self.assertIsNone(vpn._state)
state = vpn.state
mock.assert_called_once()
self.assertEqual(datetime.datetime(2019, 6, 16, 22, 13, 21),
state.up_since)
self.assertEqual('CONNECTED', state.state_name)
self.assertEqual('SUCCESS', state.desc_string)
self.assertEqual('10.0.0.1', state.local_virtual_v4_addr)
self.assertIsNone(state.remote_addr)
self.assertIsNone(state.remote_port)
self.assertEqual('1.2.3.4', state.local_addr)
self.assertEqual(1194, state.local_port)
mock.reset_mock()
_ = vpn.state
mock.assert_not_called()
def _parse_file(self, filename):
config = configparser.RawConfigParser()
logger.info('Parsing config file: %s', filename)
contents = config.read(filename)
if not contents:
raise InvalidConfigError(
'Unable to read config file: {}'.format(filename))
if 'Monitor' not in config.sections():
raise InvalidConfigError(
"'Monitor' section required, but not found in config")
# Parse 'Monitor' section
monitor = config['Monitor']
self.settings['name'] = monitor.get('name')
self.settings['geoip_data'] = monitor.get('geoip_data')
self.settings['datetime_format'] = monitor.get('datetime_format',
'%d/%m/%Y %H:%M:%S')
# Parse other (VPN) sections
for s in [s for s in config.sections() if s != 'Monitor']:
section = config[s]
# Compulsory
host = section.get('host')
try:
port = section.getint('port')
except ValueError:
port = None
socket = section.get('socket')
if (socket and host) or (socket
and port) or (not socket and not host
and not port):
raise InvalidConfigError(
'Must specify either socket or host and port')
if socket:
vpn = VPN(socket=socket)
else:
vpn = VPN(host=host, port=port)
vpn.name = s
# Optional
try:
vpn.allow_disconnect = section.getboolean(
'allow_disconnect', True)
except configparser.NoOptionError:
pass
# Add VPN
self.vpns.append(vpn)
def run(args, server):
env = new_env(args)
if args.alg == 'A3C':
trainer = A3C(env, args)
elif args.alg == 'Q':
trainer = Q(env, args)
elif args.alg == 'VPN':
env_off = new_env(args)
env_off.verbose = 0
env_off.reset()
trainer = VPN(env, args, env_off=env_off)
else:
raise ValueError('Invalid algorithm: ' + args.alg)
# Variable names that start with "local" are not saved in checkpoints.
variables_to_save = [v for v in tf.global_variables() if \
not v.name.startswith("global") and not v.name.startswith("local/target/")]
global_variables = [
v for v in tf.global_variables() if not v.name.startswith("local")
]
init_op = tf.variables_initializer(global_variables)
init_all_op = tf.global_variables_initializer()
saver = FastSaver(variables_to_save, max_to_keep=0)
var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
tf.get_variable_scope().name)
logger.info('Trainable vars:')
for v in var_list:
logger.info(' %s %s', v.name, v.get_shape())
logger.info("Num parameters: %d", trainer.local_network.num_param)
def init_fn(ses):
logger.info("Initializing all parameters.")
ses.run(init_all_op)
device = 'gpu' if args.gpu > 0 else 'cpu'
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.15)
config = tf.ConfigProto(device_filters=[
"/job:ps", "/job:worker/task:{}/{}:0".format(args.task, device)
],
gpu_options=gpu_options,
allow_soft_placement=True)
logdir = os.path.join(args.log, 'train')
summary_writer = tf.summary.FileWriter(logdir + "_%d" % args.task)
logger.info("Events directory: %s_%s", logdir, args.task)
sv = tf.train.Supervisor(
is_chief=(args.task == 0),
logdir=logdir,
saver=saver,
summary_op=None,
init_op=init_op,
init_fn=init_fn,
summary_writer=summary_writer,
ready_op=tf.report_uninitialized_variables(global_variables),
global_step=trainer.global_step,
save_model_secs=0,
save_summaries_secs=30)
logger.info(
"Starting session. If this hangs, we're mostly likely waiting to connect to the parameter server. "
+
"One common cause is that the parameter server DNS name isn't resolving yet, or is misspecified."
)
with sv.managed_session(server.target,
config=config) as sess, sess.as_default():
sess.run(trainer.sync)
trainer.start(sess, summary_writer)
global_step = sess.run(trainer.global_step)
epoch = -1
logger.info("Starting training at step=%d", global_step)
while not sv.should_stop() and (not args.max_step
or global_step < args.max_step):
if args.task == 0 and int(global_step / args.eval_freq) > epoch:
epoch = int(global_step / args.eval_freq)
filename = os.path.join(args.log, 'e%d' % (epoch))
sv.saver.save(sess, filename)
sv.saver.save(sess, os.path.join(args.log, 'latest'))
print("Saved to: %s" % filename)
trainer.process(sess)
global_step = sess.run(trainer.global_step)
if args.task == 0 and int(global_step / args.eval_freq) > epoch:
epoch = int(global_step / args.eval_freq)
filename = os.path.join(args.log, 'e%d' % (epoch))
sv.saver.save(sess, filename)
sv.saver.save(sess, os.path.join(args.log, 'latest'))
print("Saved to: %s" % filename)
# Ask for all the services to stop.
sv.stop()
logger.info('reached %s steps. worker stopped.', global_step)
def run_tester(args, server):
env = new_env(args)
# env.configure()
env.reset()
env.max_history = args.eval_num
if args.alg == 'A3C':
agent = A3C(env, args)
elif args.alg == 'Q':
agent = Q(env, args)
elif args.alg == 'VPN':
agent = VPN(env, args)
else:
raise ValueError('Invalid algorithm: ' + args.alg)
device = 'gpu' if args.gpu > 0 else 'cpu'
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.15)
config = tf.ConfigProto(device_filters=[
"/job:ps", "/job:worker/task:{}/{}:0".format(args.task, device)
],
gpu_options=gpu_options,
allow_soft_placement=True)
variables_to_save = [v for v in tf.global_variables() if \
not v.name.startswith("global") and not v.name.startswith("local/target/")]
global_variables = [
v for v in tf.global_variables() if not v.name.startswith("local")
]
init_op = tf.variables_initializer(global_variables)
init_all_op = tf.global_variables_initializer()
var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
tf.get_variable_scope().name)
logger.info('Trainable vars:')
for v in var_list:
logger.info(' %s %s', v.name, v.get_shape())
logger.info("Num parameters: %d", agent.local_network.num_param)
def init_fn(ses):
logger.info("Initializing all parameters.")
ses.run(init_all_op)
saver = FastSaver(variables_to_save, max_to_keep=0)
sv = tf.train.Supervisor(
is_chief=False,
global_step=agent.global_step,
summary_op=None,
init_op=init_op,
init_fn=init_fn,
ready_op=tf.report_uninitialized_variables(global_variables),
saver=saver,
save_model_secs=0,
save_summaries_secs=0)
best_reward = -10000
with sv.managed_session(server.target,
config=config) as sess, sess.as_default():
epoch = args.eval_epoch
while args.eval_freq * epoch <= args.max_step:
path = os.path.join(args.log, "e%d" % epoch)
if not os.path.exists(path + ".index"):
time.sleep(10)
continue
logger.info("Start evaluation (Epoch %d)", epoch)
saver.restore(sess, path)
np.random.seed(args.seed)
reward = evaluate(env,
agent.local_network,
args.eval_num,
eps=args.eps_eval)
logfile = open(os.path.join(args.log, "eval.csv"), "a")
print("Epoch: %d, Reward: %.2f" % (epoch, reward))
logfile.write("%d, %.3f\n" % (epoch, reward))
logfile.close()
if reward > best_reward:
best_reward = reward
sv.saver.save(sess, os.path.join(args.log, 'best'))
print("Saved to: %s" % os.path.join(args.log, 'best'))
epoch += 1
logger.info('tester stopped.')
def test_anchor(self):
vpn = VPN(host='localhost', port=1234)
vpn.name = 'Test VPN'
self.assertEqual(vpn.anchor, 'test_vpn')
vpn.name = 'asd_asd'
self.assertEqual(vpn.anchor, 'asd_asd')
def test_host_port_socket(self):
with self.assertRaises(MonitorError) as ctx:
VPN(host='localhost', port=1234, socket='file.sock')
self.assertEqual('Must specify either socket or host and port',
str(ctx.exception))