def test_allowed_hosts(self):
logging.debug('')
logging.debug('test_allowed_hosts')
hostname = socket.gethostname()
host_ipv4 = socket.gethostbyname(hostname)
dot = host_ipv4.rfind('.')
domain_ipv4 = host_ipv4[:dot+1]
gibberish = '$^&*'
with open('hosts.allow', 'w') as out:
out.write("""
# Local host IPv4.
%s
# Local domain IPv4.
%s
# Local host name.
%s
# Gibberish.
%s
""" % (host_ipv4, domain_ipv4, hostname, gibberish))
try:
allowed_hosts = read_allowed_hosts('hosts.allow')
finally:
os.remove('hosts.allow')
# Check read data.
expected = 3
try:
# This actally resolves in some environments.
socket.gethostbyname(gibberish)
except socket.gaierror:
pass
else:
expected += 1
self.assertEqual(len(allowed_hosts), expected)
self.assertEqual(allowed_hosts[0], host_ipv4)
self.assertEqual(allowed_hosts[1], domain_ipv4)
self.assertEqual(allowed_hosts[2], host_ipv4)
# Check AF_INET addresses.
logger = logging.getLogger()
self.assertTrue(is_legal_connection((host_ipv4, 0),
allowed_hosts, logger))
domain_host = domain_ipv4 + '123'
self.assertTrue(is_legal_connection((domain_host, 0),
allowed_hosts, logger))
self.assertFalse(is_legal_connection(('0.0.0.0', 0),
allowed_hosts, logger))
# Check AF_UNIX address.
self.assertTrue(is_legal_connection('/tmp/pipe',
allowed_hosts, logger))
def test_allowed_hosts(self):
logging.debug('')
logging.debug('test_allowed_hosts')
hostname = socket.gethostname()
host_ipv4 = socket.gethostbyname(hostname)
dot = host_ipv4.rfind('.')
domain_ipv4 = host_ipv4[:dot+1]
with open('hosts.allow', 'w') as out:
out.write("""
# Local host IPv4.
%s
# Local domain IPv4.
%s
# Local host name.
%s
# Gibberish.
$^&*
""" % (host_ipv4, domain_ipv4, hostname))
try:
allowed_hosts = read_allowed_hosts('hosts.allow')
finally:
os.remove('hosts.allow')
# Check read data.
self.assertEqual(len(allowed_hosts), 3)
self.assertEqual(allowed_hosts[0], host_ipv4)
self.assertEqual(allowed_hosts[1], domain_ipv4)
self.assertEqual(allowed_hosts[2], host_ipv4)
# Check AF_INET addresses.
logger = logging.getLogger()
self.assertTrue(is_legal_connection((host_ipv4, 0),
allowed_hosts, logger))
domain_host = domain_ipv4 + '123'
self.assertTrue(is_legal_connection((domain_host, 0),
allowed_hosts, logger))
self.assertFalse(is_legal_connection(('0.0.0.0', 0),
allowed_hosts, logger))
# Check AF_UNIX address.
self.assertTrue(is_legal_connection('/tmp/pipe',
allowed_hosts, logger))
def main(): #pragma no cover
"""
OpenMDAO factory service process.
Usage: python objserverfactory.py [--allow-public][--allow-shell][--hosts=filename][--types=filename][--users=filename][--address=address][--port=number][--prefix=name][--tunnel][--resources=filename][--log-host=hostname][--log-port=number][--log-prefix=string]
--allow-public:
Allows access by anyone from any allowed host. Use with care!
--allow-shell:
Allows access to :meth:`execute_command` and :meth:`load_model`.
Use with care!
--hosts: string
Filename for allowed hosts specification. Default ``hosts.allow``.
Ignored if '--users' is specified.
The file should contain IPv4 host addresses, IPv4 domain addresses,
or hostnames, one per line. Blank lines are ignored, and '#' marks the
start of a comment which continues to the end of the line.
For security reasons this file must be accessible only by the user
running this server.
--types: string
Filename for allowed types specification.
If not specified then allow types listed by
:meth:`factorymanager.get_available_types`.
The file should contain one type name per line.
--users: string
Filename for allowed users specification.
Ignored if '--allow-public' is specified.
Default is ``~/.ssh/authorized_keys``, other files should be of the
same format: each line has ``key-type public-key-data user@host``,
where `user` is the username on `host`. `host` will be translated to an
IPv4 address and included in the allowed hosts list.
Note that this ``user@host`` form is not necessarily enforced by
programs which generate keys.
For security reasons this file must be accessible only by the user
running this server.
--address: string
IPv4 address, hostname, or pipe name.
Default is the host's default IPv4 address.
--port: int
Server port (default of 0 implies next available port).
Note that ports below 1024 typically require special privileges.
If port is negative, then a local pipe is used for communication.
--prefix: string
Prefix for configuration and stdout/stderr files (default ``server``).
--tunnel:
Report host IP address but listen for connections from a local
SSH tunnel.
--resources: string
Filename for resource configuration. If not specified then the
default of ``~/.openmdao/resources.cfg`` will be used.
--log-host: string
Hostname to send remote log messages to.
--log-port: int
Port on `log-host` to send remote log messages to.
--log-prefix: string
Prefix to apply to remote log messages. Default is ``pid@host``.
If ``prefix.key`` exists, it is read for an authorization key string.
Otherwise public key authorization and encryption is used.
Allowed hosts *must* be specified if `port` is >= 0. Only allowed hosts
may connect to the server.
Once initialized ``prefix.cfg`` is written with address, port, and
public key information.
"""
parser = optparse.OptionParser()
parser.add_option('--address', action='store', type='str',
help='Network address to serve.')
parser.add_option('--allow-public', action='store_true', default=False,
help='Allows access by any user, use with care!')
parser.add_option('--allow-shell', action='store_true', default=False,
help='Allows potential shell access, use with care!')
parser.add_option('--hosts', action='store', type='str',
default='hosts.allow', help='Filename for allowed hosts')
parser.add_option('--types', action='store', type='str',
help='Filename for allowed types')
parser.add_option('--users', action='store', type='str',
default='~/.ssh/authorized_keys',
help='Filename for allowed users')
parser.add_option('--port', action='store', type='int', default=0,
help='Server port (0 implies next available port)')
parser.add_option('--prefix', action='store', default='server',
help='Prefix for config and stdout/stderr files')
parser.add_option('--tunnel', action='store_true', default=False,
help='Report host IP address but listen for connections'
' from a local SSH tunnel')
parser.add_option('--resources', action='store', type='str',
default=None, help='Filename for resource configuration')
parser.add_option('--log-host', action='store', type='str',
default=None, help='hostname for remote log messages')
parser.add_option('--log-port', action='store', type='int',
default=None, help='port for remote log messages')
parser.add_option('--log-prefix', action='store', type='str',
default=None, help='prefix for remote log messages')
options, arguments = parser.parse_args()
if arguments:
parser.print_help()
sys.exit(1)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
if options.log_host and options.log_port:
install_remote_handler(options.log_host, int(options.log_port),
options.log_prefix)
server_key = options.prefix+'.key'
server_cfg = options.prefix+'.cfg'
global _SERVER_CFG
_SERVER_CFG = server_cfg
# Get authkey.
authkey = 'PublicKey'
try:
with open(server_key, 'r') as inp:
authkey = inp.readline().strip()
os.remove(server_key)
except IOError:
pass
if options.allow_shell:
msg = 'Shell access is ALLOWED'
logger.warning(msg)
print msg
allowed_users = None
allowed_hosts = None
# Get allowed_users.
if options.allow_public:
allowed_users = None
msg = 'Public access is ALLOWED'
logger.warning(msg)
print msg
if options.port >= 0:
# Get allowed_hosts.
if os.path.exists(options.hosts):
try:
allowed_hosts = read_allowed_hosts(options.hosts)
except Exception as exc:
msg = "Can't read allowed hosts file %r: %s" \
% (options.hosts, exc)
logger.error(msg)
print msg
sys.exit(1)
else:
msg = 'Allowed hosts file %r does not exist.' % options.hosts
logger.error(msg)
print msg
sys.exit(1)
if not allowed_hosts:
msg = 'No hosts in allowed hosts file %r.' % options.hosts
logger.error(msg)
print msg
sys.exit(1)
else:
if os.path.exists(options.users):
try:
allowed_users = read_authorized_keys(options.users, logger)
except Exception as exc:
msg = "Can't read allowed users file %r: %s" \
% (options.users, exc)
logger.error(msg)
print msg
sys.exit(1)
else:
msg = 'Allowed users file %r does not exist.' % options.users
logger.error(msg)
print msg
sys.exit(1)
if not allowed_users:
msg = 'No users in allowed users file %r.' % options.users
logger.error(msg)
print msg
sys.exit(1)
# Get allowed_types.
allowed_types = None
if options.types:
if os.path.exists(options.types):
allowed_types = []
with open(options.types, 'r') as inp:
line = inp.readline()
while line:
line = line.strip()
if line:
allowed_types.append(line)
line = inp.readline()
else:
msg = 'Allowed types file %r does not exist.' % options.types
logger.error(msg)
print msg
sys.exit(1)
# Optionally configure resources.
if options.resources:
# Import here to avoid import loop.
from openmdao.main.resource import ResourceAllocationManager as RAM
RAM.configure(options.resources)
# Get address and create manager.
if options.port >= 0:
if options.address: # Specify IPv4/hostname.
address = (options.address, options.port)
else:
address = (platform.node(), options.port)
else:
if options.address: # Specify pipename.
address = options.address
else:
address = None
logger.info('Starting FactoryManager %s %r', address, keytype(authkey))
current_process().authkey = authkey
bind_address = ('127.0.0.1', options.port) if options.tunnel else address
manager = _FactoryManager(bind_address, authkey, name='Factory',
allowed_hosts=allowed_hosts,
allowed_users=allowed_users,
allow_tunneling=options.tunnel)
# Set defaults for created ObjServerFactories.
# There isn't a good method to propagate these through the manager.
ObjServerFactory._address = address
ObjServerFactory._allow_shell = options.allow_shell
ObjServerFactory._allowed_types = allowed_types
ObjServerFactory._allow_tunneling = options.tunnel
# Get server, retry if specified address is in use.
server = None
retries = 0
while server is None:
try:
server = manager.get_server()
except socket.error as exc:
if str(exc).find('Address already in use') >= 0:
if retries < 10:
msg = 'Address %s in use, retrying...' % (address,)
logger.debug(msg)
print msg
time.sleep(5)
retries += 1
else:
msg = 'Address %s in use, too many retries.' % (address,)
logger.error(msg)
print msg
sys.exit(1)
else:
raise
# Record configuration.
real_ip = None if address is None else address[0]
write_server_config(server, _SERVER_CFG, real_ip)
msg = 'Serving on %s' % (server.address,)
logger.info(msg)
print msg
sys.stdout.flush()
# And away we go...
signal.signal(signal.SIGTERM, _sigterm_handler)
try:
server.serve_forever()
finally:
_cleanup()
sys.exit(0)
def test_allowed_hosts(self):
logging.debug('')
logging.debug('test_allowed_hosts')
hostname = socket.gethostname()
host_ipv4 = socket.gethostbyname(hostname)
dot = host_ipv4.rfind('.')
domain_ipv4 = host_ipv4[:dot+1]
good_data = """
# Local host IPv4.
%s
# Local domain IPv4.
%s
# Local host name.
%s
""" % (host_ipv4, domain_ipv4, hostname)
gibberish = '$^&*'
bad_data = good_data + """
# Gibberish.
%s
""" % gibberish
# Try good data.
with open('hosts.allow', 'w') as out:
out.write(good_data)
if sys.platform != 'win32' or HAVE_PYWIN32:
make_private('hosts.allow')
try:
allowed_hosts = read_allowed_hosts('hosts.allow')
finally:
os.remove('hosts.allow')
self.assertEqual(len(allowed_hosts), 3)
self.assertEqual(allowed_hosts[0], host_ipv4)
self.assertEqual(allowed_hosts[1], domain_ipv4)
self.assertEqual(allowed_hosts[2], host_ipv4)
# Try bad data.
with open('hosts.allow', 'w') as out:
out.write(bad_data)
if sys.platform != 'win32' or HAVE_PYWIN32:
make_private('hosts.allow')
try:
allowed_hosts = read_allowed_hosts('hosts.allow')
except RuntimeError as exc:
self.assertEqual(str(exc),
"1 errors in 'hosts.allow', check log for details")
else:
expected = 3
try:
# This actally resolves in some environments.
socket.gethostbyname(gibberish)
except socket.gaierror:
pass
else:
expected += 1
self.assertEqual(len(allowed_hosts), expected)
self.assertEqual(allowed_hosts[0], host_ipv4)
self.assertEqual(allowed_hosts[1], domain_ipv4)
self.assertEqual(allowed_hosts[2], host_ipv4)
finally:
os.remove('hosts.allow')
# Check AF_INET addresses.
logger = logging.getLogger()
self.assertTrue(is_legal_connection((host_ipv4, 0),
allowed_hosts, logger))
domain_host = domain_ipv4 + '123'
self.assertTrue(is_legal_connection((domain_host, 0),
allowed_hosts, logger))
self.assertFalse(is_legal_connection(('0.0.0.0', 0),
allowed_hosts, logger))
# Check AF_UNIX address.
self.assertTrue(is_legal_connection('/tmp/pipe',
allowed_hosts, logger))
# Try nonexistant file.
assert_raises(self, "read_allowed_hosts('no-such-file')",
globals(), locals(), RuntimeError,
"'no-such-file' does not exist")
# Try insecure file.
if sys.platform != 'win32' or HAVE_PYWIN32:
with open('hosts.allow', 'w') as out:
out.write('\n')
os.chmod('hosts.allow', 0666)
try:
assert_raises(self, "read_allowed_hosts('hosts.allow')",
globals(), locals(), RuntimeError,
"'hosts.allow' is not private")
finally:
os.remove('hosts.allow')
def main(): # pragma no cover
"""
OpenMDAO factory service process.
Usage: python objserverfactory.py [--allow-public][--allow-shell][--hosts=filename][--types=filename][--users=filename][--address=address][--port=number][--prefix=name][--tunnel][--resources=filename][--log-host=hostname][--log-port=number][--log-prefix=string]
--allow-public:
Allows access by anyone from any allowed host. Use with care!
--allow-shell:
Allows access to :meth:`execute_command` and :meth:`load_model`.
Use with care!
--hosts: string
Filename for allowed hosts specification. Default ``hosts.allow``.
Ignored if '--users' is specified.
The file should contain IPv4 host addresses, IPv4 domain addresses,
or hostnames, one per line. Blank lines are ignored, and '#' marks the
start of a comment which continues to the end of the line.
For security reasons this file must be accessible only by the user
running this server.
--types: string
Filename for allowed types specification.
If not specified then allow types listed by
:meth:`factorymanager.get_available_types`.
The file should contain one type name per line.
--users: string
Filename for allowed users specification.
Ignored if '--allow-public' is specified.
Default is ``~/.ssh/authorized_keys``, other files should be of the
same format: each line has ``key-type public-key-data user@host``,
where `user` is the username on `host`. `host` will be translated to an
IPv4 address and included in the allowed hosts list.
Note that this ``user@host`` form is not necessarily enforced by
programs which generate keys.
For security reasons this file must be accessible only by the user
running this server.
--address: string
IPv4 address, hostname, or pipe name.
Default is the host's default IPv4 address.
--port: int
Server port (default of 0 implies next available port).
Note that ports below 1024 typically require special privileges.
If port is negative, then a local pipe is used for communication.
--prefix: string
Prefix for configuration and stdout/stderr files (default ``server``).
--tunnel:
Report host IP address but listen for connections from a local
SSH tunnel.
--resources: string
Filename for resource configuration. If not specified then the
default of ``~/.openmdao/resources.cfg`` will be used.
--log-host: string
Hostname to send remote log messages to.
--log-port: int
Port on `log-host` to send remote log messages to.
--log-prefix: string
Prefix to apply to remote log messages. Default is ``pid@host``.
If ``prefix.key`` exists, it is read for an authorization key string.
Otherwise, public key authorization and encryption is used.
Allowed hosts *must* be specified if `port` is >= 0. Only allowed hosts
may connect to the server.
Once initialized ``prefix.cfg`` is written with address, port, and
public key information.
"""
parser = optparse.OptionParser()
parser.add_option('--address',
action='store',
type='str',
help='Network address to serve.')
parser.add_option('--allow-public',
action='store_true',
default=False,
help='Allows access by any user, use with care!')
parser.add_option('--allow-shell',
action='store_true',
default=False,
help='Allows potential shell access, use with care!')
parser.add_option('--hosts',
action='store',
type='str',
default='hosts.allow',
help='Filename for allowed hosts')
parser.add_option('--types',
action='store',
type='str',
help='Filename for allowed types')
parser.add_option('--users',
action='store',
type='str',
default='~/.ssh/authorized_keys',
help='Filename for allowed users')
parser.add_option('--port',
action='store',
type='int',
default=0,
help='Server port (0 implies next available port)')
parser.add_option('--prefix',
action='store',
default='server',
help='Prefix for config and stdout/stderr files')
parser.add_option('--tunnel',
action='store_true',
default=False,
help='Report host IP address but listen for connections'
' from a local SSH tunnel')
parser.add_option('--resources',
action='store',
type='str',
default=None,
help='Filename for resource configuration')
parser.add_option('--log-host',
action='store',
type='str',
default=None,
help='hostname for remote log messages')
parser.add_option('--log-port',
action='store',
type='int',
default=None,
help='port for remote log messages')
parser.add_option('--log-prefix',
action='store',
type='str',
default=None,
help='prefix for remote log messages')
options, arguments = parser.parse_args()
if arguments:
parser.print_help()
sys.exit(1)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
if options.log_host and options.log_port:
install_remote_handler(options.log_host, int(options.log_port),
options.log_prefix)
server_key = options.prefix + '.key'
server_cfg = options.prefix + '.cfg'
global _SERVER_CFG
_SERVER_CFG = server_cfg
# Get authkey.
authkey = 'PublicKey'
try:
with open(server_key, 'r') as inp:
authkey = inp.readline().strip()
os.remove(server_key)
except IOError:
pass
if options.allow_shell:
msg = 'Shell access is ALLOWED'
logger.warning(msg)
print msg
allowed_users = None
allowed_hosts = None
# Get allowed_users.
if options.allow_public:
allowed_users = None
msg = 'Public access is ALLOWED'
logger.warning(msg)
print msg
if options.port >= 0:
# Get allowed_hosts.
if os.path.exists(options.hosts):
try:
allowed_hosts = read_allowed_hosts(options.hosts)
except Exception as exc:
msg = "Can't read allowed hosts file %r: %s" \
% (options.hosts, exc)
logger.error(msg)
print msg
sys.exit(1)
else:
msg = 'Allowed hosts file %r does not exist.' % options.hosts
logger.error(msg)
print msg
sys.exit(1)
if not allowed_hosts:
msg = 'No hosts in allowed hosts file %r.' % options.hosts
logger.error(msg)
print msg
sys.exit(1)
else:
if os.path.exists(options.users):
try:
allowed_users = read_authorized_keys(options.users, logger)
except Exception as exc:
msg = "Can't read allowed users file %r: %s" \
% (options.users, exc)
logger.error(msg)
print msg
sys.exit(1)
else:
msg = 'Allowed users file %r does not exist.' % options.users
logger.error(msg)
print msg
sys.exit(1)
if not allowed_users:
msg = 'No users in allowed users file %r.' % options.users
logger.error(msg)
print msg
sys.exit(1)
# Get allowed_types.
allowed_types = None
if options.types:
if os.path.exists(options.types):
allowed_types = []
with open(options.types, 'r') as inp:
line = inp.readline()
while line:
line = line.strip()
if line:
allowed_types.append(line)
line = inp.readline()
else:
msg = 'Allowed types file %r does not exist.' % options.types
logger.error(msg)
print msg
sys.exit(1)
# Optionally configure resources.
if options.resources:
# Import here to avoid import loop.
from openmdao.main.resource import ResourceAllocationManager as RAM
RAM.configure(options.resources)
# Get address and create manager.
if options.port >= 0:
if options.address: # Specify IPv4/hostname.
address = (options.address, options.port)
else:
address = (platform.node(), options.port)
else:
if options.address: # Specify pipename.
address = options.address
else:
address = None
logger.info('Starting FactoryManager %s %r', address, keytype(authkey))
current_process().authkey = authkey
bind_address = ('127.0.0.1', options.port) if options.tunnel else address
manager = _FactoryManager(bind_address,
authkey,
name='Factory',
allowed_hosts=allowed_hosts,
allowed_users=allowed_users,
allow_tunneling=options.tunnel)
# Set defaults for created ObjServerFactories.
# There isn't a good method to propagate these through the manager.
ObjServerFactory._address = address
ObjServerFactory._allow_shell = options.allow_shell
ObjServerFactory._allowed_types = allowed_types
ObjServerFactory._allow_tunneling = options.tunnel
# Get server, retry if specified address is in use.
server = None
retries = 0
while server is None:
try:
server = manager.get_server()
except socket.error as exc:
if str(exc).find('Address already in use') >= 0:
if retries < 10:
msg = 'Address %s in use, retrying...' % (address, )
logger.debug(msg)
print msg
time.sleep(5)
retries += 1
else:
msg = 'Address %s in use, too many retries.' % (address, )
logger.error(msg)
print msg
sys.exit(1)
else:
raise
# Record configuration.
real_ip = None if address is None else address[0]
write_server_config(server, _SERVER_CFG, real_ip)
msg = 'Serving on %s' % (server.address, )
logger.info(msg)
print msg
sys.stdout.flush()
# And away we go...
signal.signal(signal.SIGTERM, _sigterm_handler)
try:
server.serve_forever()
finally:
_cleanup()
sys.exit(0)
def test_allowed_hosts(self):
logging.debug('')
logging.debug('test_allowed_hosts')
hostname = socket.gethostname()
host_ipv4 = socket.gethostbyname(hostname)
dot = host_ipv4.rfind('.')
domain_ipv4 = host_ipv4[:dot + 1]
good_data = """
# Local host IPv4.
%s
# Local domain IPv4.
%s
# Local host name.
%s
""" % (host_ipv4, domain_ipv4, hostname)
gibberish = '$^&*'
bad_data = good_data + """
# Gibberish.
%s
""" % gibberish
# Try good data.
with open('hosts.allow', 'w') as out:
out.write(good_data)
if sys.platform != 'win32' or HAVE_PYWIN32:
make_private('hosts.allow')
try:
allowed_hosts = read_allowed_hosts('hosts.allow')
finally:
os.remove('hosts.allow')
self.assertEqual(len(allowed_hosts), 3)
self.assertEqual(allowed_hosts[0], host_ipv4)
self.assertEqual(allowed_hosts[1], domain_ipv4)
self.assertEqual(allowed_hosts[2], host_ipv4)
# Try bad data.
with open('hosts.allow', 'w') as out:
out.write(bad_data)
if sys.platform != 'win32' or HAVE_PYWIN32:
make_private('hosts.allow')
try:
allowed_hosts = read_allowed_hosts('hosts.allow')
except RuntimeError as exc:
self.assertEqual(
str(exc), "1 errors in 'hosts.allow', check log for details")
else:
expected = 3
try:
# This actally resolves in some environments.
socket.gethostbyname(gibberish)
except socket.gaierror:
pass
else:
expected += 1
self.assertEqual(len(allowed_hosts), expected)
self.assertEqual(allowed_hosts[0], host_ipv4)
self.assertEqual(allowed_hosts[1], domain_ipv4)
self.assertEqual(allowed_hosts[2], host_ipv4)
finally:
os.remove('hosts.allow')
# Check AF_INET addresses.
logger = logging.getLogger()
self.assertTrue(
is_legal_connection((host_ipv4, 0), allowed_hosts, logger))
domain_host = domain_ipv4 + '123'
self.assertTrue(
is_legal_connection((domain_host, 0), allowed_hosts, logger))
self.assertFalse(
is_legal_connection(('0.0.0.0', 0), allowed_hosts, logger))
# Check AF_UNIX address.
self.assertTrue(is_legal_connection('/tmp/pipe', allowed_hosts,
logger))
# Try nonexistant file.
assert_raises(self, "read_allowed_hosts('no-such-file')", globals(),
locals(), RuntimeError, "'no-such-file' does not exist")
# Try insecure file.
if sys.platform != 'win32' or HAVE_PYWIN32:
with open('hosts.allow', 'w') as out:
out.write('\n')
os.chmod('hosts.allow', 0666)
try:
assert_raises(self, "read_allowed_hosts('hosts.allow')",
globals(), locals(), RuntimeError,
"'hosts.allow' is not private")
finally:
os.remove('hosts.allow')
