def start_volttron_process(opts):
'''Start the main volttron process.
Typically this function is used from main.py and just uses the argparser's
Options arguments as inputs. It also can be called with a dictionary. In
that case the dictionaries keys are mapped into a value that acts like the
args options.
'''
if isinstance(opts, dict):
opts = type('Options', (), opts)()
# vip_address is meant to be a list so make it so.
if not isinstance(opts.vip_address, list):
opts.vip_address = [opts.vip_address]
if opts.log:
opts.log = config.expandall(opts.log)
if opts.log_config:
opts.log_config = config.expandall(opts.log_config)
# Configure logging
level = max(1, opts.verboseness)
if opts.monitor and level > logging.INFO:
level = logging.INFO
if opts.log is None:
log_to_file(sys.stderr, level)
elif opts.log == '-':
log_to_file(sys.stdout, level)
elif opts.log:
log_to_file(opts.log, level, handler_class=handlers.WatchedFileHandler)
else:
log_to_file(None, 100, handler_class=lambda x: logging.NullHandler())
if opts.log_config:
with open(opts.log_config, 'r') as f:
for line in f.readlines():
_log.info(line.rstrip())
error = configure_logging(opts.log_config)
if error:
_log.error('{}: {}'.format(*error))
sys.exit(1)
opts.publish_address = config.expandall(opts.publish_address)
opts.subscribe_address = config.expandall(opts.subscribe_address)
opts.vip_address = [config.expandall(addr) for addr in opts.vip_address]
opts.vip_local_address = config.expandall(opts.vip_local_address)
if opts.instance_name is None:
if len(opts.vip_address) > 0:
opts.instance_name = opts.vip_address[0]
import urlparse
if opts.bind_web_address:
parsed = urlparse.urlparse(opts.bind_web_address)
if parsed.scheme not in ('http', 'https'):
raise StandardError(
'bind-web-address must begin with http or https.')
opts.bind_web_address = config.expandall(opts.bind_web_address)
if opts.volttron_central_address:
parsed = urlparse.urlparse(opts.volttron_central_address)
if parsed.scheme not in ('http', 'https', 'tcp'):
raise StandardError(
'volttron-central-address must begin with tcp, http or https.')
opts.volttron_central_address = config.expandall(
opts.volttron_central_address)
opts.volttron_central_serverkey = opts.volttron_central_serverkey
# Log configuration options
if getattr(opts, 'show_config', False):
_log.info('volttron version: {}'.format(__version__))
for name, value in sorted(vars(opts).iteritems()):
_log.info("%s: %s" % (name, str(repr(value))))
# Increase open files resource limit to max or 8192 if unlimited
try:
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
except OSError:
_log.exception('error getting open file limits')
else:
if soft != hard and soft != resource.RLIM_INFINITY:
try:
limit = 8192 if hard == resource.RLIM_INFINITY else hard
resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
except OSError:
_log.exception('error setting open file limits')
else:
_log.debug('open file resource limit increased from %d to %d',
soft, limit)
_log.debug('open file resource limit %d to %d', soft, hard)
# Set configuration
if HAVE_RESTRICTED:
if opts.verify_agents:
_log.info('Agent integrity verification enabled')
if opts.resource_monitor:
_log.info('Resource monitor enabled')
opts.resmon = resmon.ResourceMonitor()
opts.aip = aip.AIPplatform(opts)
opts.aip.setup()
# Check for secure mode/permissions on VOLTTRON_HOME directory
mode = os.stat(opts.volttron_home).st_mode
if mode & (stat.S_IWGRP | stat.S_IWOTH):
_log.warning('insecure mode on directory: %s', opts.volttron_home)
# Get or generate encryption key
keystore = KeyStore()
_log.debug('using key-store file %s', keystore.filename)
if not keystore.isvalid():
_log.warning('key store is invalid; connections may fail')
st = os.stat(keystore.filename)
if st.st_mode & (stat.S_IRWXG | stat.S_IRWXO):
_log.warning('insecure mode on key file')
publickey = decode_key(keystore.public)
if publickey:
_log.info('public key: %s', encode_key(publickey))
# Authorize the platform key:
entry = AuthEntry(credentials=encode_key(publickey),
user_id='platform',
comments='Automatically added by platform on start')
AuthFile().add(entry, overwrite=True)
# Add platform key to known-hosts file:
known_hosts = KnownHostsStore()
known_hosts.add(opts.vip_local_address, encode_key(publickey))
for addr in opts.vip_address:
known_hosts.add(addr, encode_key(publickey))
secretkey = decode_key(keystore.secret)
# The following line doesn't appear to do anything, but it creates
# a context common to the green and non-green zmq modules.
zmq.Context.instance() # DO NOT REMOVE LINE!!
# zmq.Context.instance().set(zmq.MAX_SOCKETS, 2046)
tracker = Tracker()
protected_topics_file = os.path.join(opts.volttron_home,
'protected_topics.json')
_log.debug('protected topics file %s', protected_topics_file)
external_address_file = os.path.join(opts.volttron_home,
'external_address.json')
_log.debug('external_address_file file %s', external_address_file)
protected_topics = {}
# Main loops
def router(stop):
try:
Router(opts.vip_local_address,
opts.vip_address,
secretkey=secretkey,
publickey=publickey,
default_user_id=b'vip.service',
monitor=opts.monitor,
tracker=tracker,
volttron_central_address=opts.volttron_central_address,
volttron_central_serverkey=opts.volttron_central_serverkey,
instance_name=opts.instance_name,
bind_web_address=opts.bind_web_address,
protected_topics=protected_topics,
external_address_file=external_address_file,
msgdebug=opts.msgdebug).run()
except Exception:
_log.exception('Unhandled exception in router loop')
raise
finally:
stop()
address = 'inproc://vip'
try:
# Start the config store before auth so we may one day have auth use it.
config_store = ConfigStoreService(address=address,
identity=CONFIGURATION_STORE)
event = gevent.event.Event()
config_store_task = gevent.spawn(config_store.core.run, event)
event.wait()
del event
# Ensure auth service is running before router
auth_file = os.path.join(opts.volttron_home, 'auth.json')
auth = AuthService(auth_file,
protected_topics_file,
opts.setup_mode,
opts.aip,
address=address,
identity=AUTH,
enable_store=False)
event = gevent.event.Event()
auth_task = gevent.spawn(auth.core.run, event)
event.wait()
del event
protected_topics = auth.get_protected_topics()
_log.debug(
"MAIN: protected topics content {}".format(protected_topics))
# Start router in separate thread to remain responsive
thread = threading.Thread(target=router, args=(auth.core.stop, ))
thread.daemon = True
thread.start()
gevent.sleep(0.1)
if not thread.isAlive():
sys.exit()
# The instance file is where we are going to record the instance and
# its details according to
instance_file = os.path.expanduser('~/.volttron_instances')
try:
instances = load_create_store(instance_file)
except ValueError:
os.remove(instance_file)
instances = load_create_store(instance_file)
this_instance = instances.get(opts.volttron_home, {})
this_instance['pid'] = os.getpid()
this_instance['version'] = __version__
# note vip_address is a list
this_instance['vip-address'] = opts.vip_address
this_instance['volttron-home'] = opts.volttron_home
this_instance['volttron-root'] = os.path.abspath('../..')
this_instance['start-args'] = sys.argv[1:]
instances[opts.volttron_home] = this_instance
instances.async_sync()
protected_topics_file = os.path.join(opts.volttron_home,
'protected_topics.json')
_log.debug('protected topics file %s', protected_topics_file)
external_address_file = os.path.join(opts.volttron_home,
'external_address.json')
_log.debug('external_address_file file %s', external_address_file)
# Launch additional services and wait for them to start before
# auto-starting agents
services = [
ControlService(opts.aip,
address=address,
identity='control',
tracker=tracker,
heartbeat_autostart=True,
enable_store=False,
enable_channel=True),
CompatPubSub(address=address,
identity='pubsub.compat',
publish_address=opts.publish_address,
subscribe_address=opts.subscribe_address),
MasterWebService(
serverkey=publickey,
identity=MASTER_WEB,
address=address,
bind_web_address=opts.bind_web_address,
volttron_central_address=opts.volttron_central_address,
aip=opts.aip,
enable_store=False),
KeyDiscoveryAgent(address=address,
serverkey=publickey,
identity='keydiscovery',
external_address_config=external_address_file,
setup_mode=opts.setup_mode,
bind_web_address=opts.bind_web_address),
PubSubWrapper(address=address,
identity='pubsub',
heartbeat_autostart=True,
enable_store=False)
]
events = [gevent.event.Event() for service in services]
tasks = [
gevent.spawn(service.core.run, event)
for service, event in zip(services, events)
]
tasks.append(config_store_task)
tasks.append(auth_task)
gevent.wait(events)
del events
# Auto-start agents now that all services are up
if opts.autostart:
for name, error in opts.aip.autostart():
_log.error('error starting {!r}: {}\n'.format(name, error))
# Wait for any service to stop, signaling exit
try:
gevent.wait(tasks, count=1)
except KeyboardInterrupt:
_log.info('SIGINT received; shutting down')
sys.stderr.write('Shutting down.\n')
for task in tasks:
task.kill(block=False)
gevent.wait(tasks)
finally:
opts.aip.finish()
def main(argv=sys.argv):
volttron_home = config.expandall(
os.environ.get('VOLTTRON_HOME', '~/.volttron'))
os.environ['VOLTTRON_HOME'] = volttron_home
# Setup option parser
parser = config.ArgumentParser(
prog=os.path.basename(argv[0]),
add_help=False,
description='VOLTTRON platform service',
usage='%(prog)s [OPTION]...',
argument_default=argparse.SUPPRESS,
epilog='Boolean options, which take no argument, may be inversed by '
'prefixing the option with no- (e.g. --autostart may be '
'inversed using --no-autostart).')
parser.add_argument('-c',
'--config',
metavar='FILE',
action='parse_config',
ignore_unknown=True,
sections=[None, 'volttron'],
help='read configuration from FILE')
parser.add_argument('-l',
'--log',
metavar='FILE',
default=None,
help='send log output to FILE instead of stderr')
parser.add_argument('-L',
'--log-config',
metavar='FILE',
help='read logging configuration from FILE')
parser.add_argument('--log-level',
metavar='LOGGER:LEVEL',
action=LogLevelAction,
help='override default logger logging level')
parser.add_argument(
'-q',
'--quiet',
action='add_const',
const=10,
dest='verboseness',
help='decrease logger verboseness; may be used multiple times')
parser.add_argument(
'-v',
'--verbose',
action='add_const',
const=-10,
dest='verboseness',
help='increase logger verboseness; may be used multiple times')
parser.add_argument('--verboseness',
type=int,
metavar='LEVEL',
default=logging.WARNING,
help='set logger verboseness')
#parser.add_argument(
# '--volttron-home', env_var='VOLTTRON_HOME', metavar='PATH',
# help='VOLTTRON configuration directory')
parser.add_argument('--show-config',
action='store_true',
help=argparse.SUPPRESS)
parser.add_help_argument()
parser.add_version_argument(version='%(prog)s ' + __version__)
agents = parser.add_argument_group('agent options')
agents.add_argument('--autostart',
action='store_true',
inverse='--no-autostart',
help='automatically start enabled agents and services')
agents.add_argument('--no-autostart',
action='store_false',
dest='autostart',
help=argparse.SUPPRESS)
agents.add_argument('--publish-address',
metavar='ZMQADDR',
help='ZeroMQ URL used for agent publishing')
agents.add_argument('--subscribe-address',
metavar='ZMQADDR',
help='ZeroMQ URL used for agent subscriptions')
agents.add_argument('--vip-address',
metavar='ZMQADDR',
action='append',
default=[],
help='ZeroMQ URL to bind for VIP connections')
# XXX: re-implement control options
#on
#control.add_argument(
# '--allow-root', action='store_true', inverse='--no-allow-root',
# help='allow root to connect to control socket')
#control.add_argument(
# '--no-allow-root', action='store_false', dest='allow_root',
# help=argparse.SUPPRESS)
#control.add_argument(
# '--allow-users', action='store_list', metavar='LIST',
# help='users allowed to connect to control socket')
#control.add_argument(
# '--allow-groups', action='store_list', metavar='LIST',
# help='user groups allowed to connect to control socket')
if HAVE_RESTRICTED:
class RestrictedAction(argparse.Action):
def __init__(self,
option_strings,
dest,
const=True,
help=None,
**kwargs):
super(RestrictedAction, self).__init__(option_strings,
dest=argparse.SUPPRESS,
nargs=0,
const=const,
help=help)
def __call__(self, parser, namespace, values, option_string=None):
namespace.verify_agents = self.const
namespace.resource_monitor = self.const
namespace.mobility = self.const
restrict = parser.add_argument_group('restricted options')
restrict.add_argument(
'--restricted',
action=RestrictedAction,
inverse='--no-restricted',
help='shortcut to enable all restricted features')
restrict.add_argument('--no-restricted',
action=RestrictedAction,
const=False,
help=argparse.SUPPRESS)
restrict.add_argument('--verify',
action='store_true',
inverse='--no-verify',
help='verify agent integrity before execution')
restrict.add_argument('--no-verify',
action='store_false',
dest='verify_agents',
help=argparse.SUPPRESS)
restrict.add_argument('--resource-monitor',
action='store_true',
inverse='--no-resource-monitor',
help='enable agent resource management')
restrict.add_argument('--no-resource-monitor',
action='store_false',
dest='resource_monitor',
help=argparse.SUPPRESS)
restrict.add_argument('--mobility',
action='store_true',
inverse='--no-mobility',
help='enable agent mobility')
restrict.add_argument('--no-mobility',
action='store_false',
dest='mobility',
help=argparse.SUPPRESS)
restrict.add_argument('--mobility-address',
metavar='ADDRESS',
help='specify the address on which to listen')
restrict.add_argument('--mobility-port',
type=int,
metavar='NUMBER',
help='specify the port on which to listen')
vip_path = '$VOLTTRON_HOME/run/vip.socket'
if sys.platform.startswith('linux'):
vip_path = '@' + vip_path
parser.set_defaults(
log=None,
log_config=None,
verboseness=logging.WARNING,
volttron_home=volttron_home,
autostart=True,
publish_address='ipc://$VOLTTRON_HOME/run/publish',
subscribe_address='ipc://$VOLTTRON_HOME/run/subscribe',
vip_address=['ipc://' + vip_path],
#allow_root=False,
#allow_users=None,
#allow_groups=None,
verify_agents=True,
resource_monitor=True,
mobility=True,
mobility_address=None,
mobility_port=2522)
# Parse and expand options
args = argv[1:]
conf = os.path.join(volttron_home, 'config')
if os.path.exists(conf) and 'SKIP_VOLTTRON_CONFIG' not in os.environ:
args = ['--config', conf] + args
logging.getLogger().setLevel(logging.NOTSET)
opts = parser.parse_args(args)
if opts.log:
opts.log = config.expandall(opts.log)
if opts.log_config:
opts.log_config = config.expandall(opts.log_config)
opts.publish_address = config.expandall(opts.publish_address)
opts.subscribe_address = config.expandall(opts.subscribe_address)
opts.vip_address = [config.expandall(addr) for addr in opts.vip_address]
if HAVE_RESTRICTED:
# Set mobility defaults
if opts.mobility_address is None:
info = socket.getaddrinfo(None, 0, 0, socket.SOCK_STREAM, 0,
socket.AI_NUMERICHOST)
family = info[0][0] if info else ''
opts.mobility_address = '::' if family == socket.AF_INET6 else ''
if getattr(opts, 'show_config', False):
for name, value in sorted(vars(opts).iteritems()):
print(name, repr(value))
return
# Configure logging
level = max(1, opts.verboseness)
if opts.log is None:
log_to_file(sys.stderr, level)
elif opts.log == '-':
log_to_file(sys.stdout, level)
elif opts.log:
log_to_file(opts.log, level, handler_class=handlers.WatchedFileHandler)
else:
log_to_file(None, 100, handler_class=lambda x: logging.NullHandler())
if opts.log_config:
error = configure_logging(opts.log_config)
if error:
parser.error('{}: {}'.format(*error))
# Setup mobility server
if HAVE_RESTRICTED and opts.mobility:
ssh_dir = os.path.join(opts.volttron_home, 'ssh')
try:
priv_key = RSAKey(filename=os.path.join(ssh_dir, 'id_rsa'))
authorized_keys = comms.load_authorized_keys(
os.path.join(ssh_dir, 'authorized_keys'))
except (OSError, IOError, PasswordRequiredException,
SSHException) as exc:
parser.error(exc)
# Set configuration
if HAVE_RESTRICTED:
if opts.verify_agents:
_log.info('Agent integrity verification enabled')
if opts.resource_monitor:
_log.info('Resource monitor enabled')
opts.resmon = resmon.ResourceMonitor()
opts.aip = aip.AIPplatform(opts)
opts.aip.setup()
if opts.autostart:
for name, error in opts.aip.autostart():
_log.error('error starting {!r}: {}\n'.format(name, error))
# Main loops
try:
router = Router(opts.vip_address)
exchange = gevent.spawn(agent_exchange, opts.publish_address,
opts.subscribe_address)
control = gevent.spawn(
ControlService(opts.aip,
vip_address='inproc://vip',
vip_identity='control').run)
pubsub = gevent.spawn(
PubSubService(vip_address='inproc://vip',
vip_identity='pubsub').run)
if HAVE_RESTRICTED and opts.mobility:
address = (opts.mobility_address, opts.mobility_port)
mobility_in = comms_server.ThreadedServer(address, priv_key,
authorized_keys,
opts.aip)
mobility_in.start()
mobility_out = MobilityAgent(
opts.aip,
subscribe_address=opts.subscribe_address,
publish_address=opts.publish_address)
gevent.spawn(mobility_out.run)
try:
router.run()
finally:
control.kill()
pubsub.kill()
exchange.kill()
finally:
opts.aip.finish()
def main(argv=sys.argv):
# Refuse to run as root
if not getattr(os, 'getuid', lambda: -1)():
sys.stderr.write('%s: error: refusing to run as root to prevent '
'potential damage.\n' % os.path.basename(argv[0]))
sys.exit(77)
volttron_home = os.path.normpath(
config.expandall(os.environ.get('VOLTTRON_HOME', '~/.volttron')))
os.environ['VOLTTRON_HOME'] = volttron_home
# Setup option parser
parser = config.ArgumentParser(
prog=os.path.basename(argv[0]),
add_help=False,
description='VOLTTRON platform service',
usage='%(prog)s [OPTION]...',
argument_default=argparse.SUPPRESS,
epilog='Boolean options, which take no argument, may be inversed by '
'prefixing the option with no- (e.g. --autostart may be '
'inversed using --no-autostart).')
parser.add_argument('-c',
'--config',
metavar='FILE',
action='parse_config',
ignore_unknown=True,
sections=[None, 'volttron'],
help='read configuration from FILE')
parser.add_argument('-l',
'--log',
metavar='FILE',
default=None,
help='send log output to FILE instead of stderr')
parser.add_argument('-L',
'--log-config',
metavar='FILE',
help='read logging configuration from FILE')
parser.add_argument('--log-level',
metavar='LOGGER:LEVEL',
action=LogLevelAction,
help='override default logger logging level')
parser.add_argument('--monitor',
action='store_true',
help='monitor and log connections (implies -v)')
parser.add_argument(
'-q',
'--quiet',
action='add_const',
const=10,
dest='verboseness',
help='decrease logger verboseness; may be used multiple times')
parser.add_argument(
'-v',
'--verbose',
action='add_const',
const=-10,
dest='verboseness',
help='increase logger verboseness; may be used multiple times')
parser.add_argument('--verboseness',
type=int,
metavar='LEVEL',
default=logging.WARNING,
help='set logger verboseness')
#parser.add_argument(
# '--volttron-home', env_var='VOLTTRON_HOME', metavar='PATH',
# help='VOLTTRON configuration directory')
parser.add_argument('--show-config',
action='store_true',
help=argparse.SUPPRESS)
parser.add_help_argument()
parser.add_version_argument(version='%(prog)s ' + __version__)
agents = parser.add_argument_group('agent options')
agents.add_argument('--autostart',
action='store_true',
inverse='--no-autostart',
help='automatically start enabled agents and services')
agents.add_argument('--no-autostart',
action='store_false',
dest='autostart',
help=argparse.SUPPRESS)
agents.add_argument(
'--publish-address',
metavar='ZMQADDR',
help='ZeroMQ URL used for pre-3.x agent publishing (deprecated)')
agents.add_argument(
'--subscribe-address',
metavar='ZMQADDR',
help='ZeroMQ URL used for pre-3.x agent subscriptions (deprecated)')
agents.add_argument('--vip-address',
metavar='ZMQADDR',
action='append',
default=[],
help='ZeroMQ URL to bind for VIP connections')
agents.add_argument(
'--vip-local-address',
metavar='ZMQADDR',
help='ZeroMQ URL to bind for local agent VIP connections')
# XXX: re-implement control options
#on
#control.add_argument(
# '--allow-root', action='store_true', inverse='--no-allow-root',
# help='allow root to connect to control socket')
#control.add_argument(
# '--no-allow-root', action='store_false', dest='allow_root',
# help=argparse.SUPPRESS)
#control.add_argument(
# '--allow-users', action='store_list', metavar='LIST',
# help='users allowed to connect to control socket')
#control.add_argument(
# '--allow-groups', action='store_list', metavar='LIST',
# help='user groups allowed to connect to control socket')
if HAVE_RESTRICTED:
class RestrictedAction(argparse.Action):
def __init__(self,
option_strings,
dest,
const=True,
help=None,
**kwargs):
super(RestrictedAction, self).__init__(option_strings,
dest=argparse.SUPPRESS,
nargs=0,
const=const,
help=help)
def __call__(self, parser, namespace, values, option_string=None):
namespace.verify_agents = self.const
namespace.resource_monitor = self.const
#namespace.mobility = self.const
restrict = parser.add_argument_group('restricted options')
restrict.add_argument(
'--restricted',
action=RestrictedAction,
inverse='--no-restricted',
help='shortcut to enable all restricted features')
restrict.add_argument('--no-restricted',
action=RestrictedAction,
const=False,
help=argparse.SUPPRESS)
restrict.add_argument('--verify',
action='store_true',
inverse='--no-verify',
help='verify agent integrity before execution')
restrict.add_argument('--no-verify',
action='store_false',
dest='verify_agents',
help=argparse.SUPPRESS)
restrict.add_argument('--resource-monitor',
action='store_true',
inverse='--no-resource-monitor',
help='enable agent resource management')
restrict.add_argument('--no-resource-monitor',
action='store_false',
dest='resource_monitor',
help=argparse.SUPPRESS)
#restrict.add_argument(
# '--mobility', action='store_true', inverse='--no-mobility',
# help='enable agent mobility')
#restrict.add_argument(
# '--no-mobility', action='store_false', dest='mobility',
# help=argparse.SUPPRESS)
ipc = 'ipc://%s$VOLTTRON_HOME/run/' % (
'@' if sys.platform.startswith('linux') else '')
parser.set_defaults(
log=None,
log_config=None,
monitor=False,
verboseness=logging.WARNING,
volttron_home=volttron_home,
autostart=True,
publish_address=ipc + 'publish',
subscribe_address=ipc + 'subscribe',
vip_address=[],
vip_local_address=ipc + 'vip.socket',
#allow_root=False,
#allow_users=None,
#allow_groups=None,
verify_agents=True,
resource_monitor=True,
#mobility=True,
)
# Parse and expand options
args = argv[1:]
conf = os.path.join(volttron_home, 'config')
if os.path.exists(conf) and 'SKIP_VOLTTRON_CONFIG' not in os.environ:
args = ['--config', conf] + args
logging.getLogger().setLevel(logging.NOTSET)
opts = parser.parse_args(args)
if opts.log:
opts.log = config.expandall(opts.log)
if opts.log_config:
opts.log_config = config.expandall(opts.log_config)
opts.publish_address = config.expandall(opts.publish_address)
opts.subscribe_address = config.expandall(opts.subscribe_address)
opts.vip_address = [config.expandall(addr) for addr in opts.vip_address]
opts.vip_local_address = config.expandall(opts.vip_local_address)
if getattr(opts, 'show_config', False):
for name, value in sorted(vars(opts).iteritems()):
print(name, repr(value))
return
# Configure logging
level = max(1, opts.verboseness)
if opts.monitor and level > logging.INFO:
level = logging.INFO
if opts.log is None:
log_to_file(sys.stderr, level)
elif opts.log == '-':
log_to_file(sys.stdout, level)
elif opts.log:
log_to_file(opts.log, level, handler_class=handlers.WatchedFileHandler)
else:
log_to_file(None, 100, handler_class=lambda x: logging.NullHandler())
if opts.log_config:
error = configure_logging(opts.log_config)
if error:
parser.error('{}: {}'.format(*error))
# Increase open files resource limit to max or 8192 if unlimited
try:
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
except OSError:
_log.exception('error getting open file limits')
else:
if soft != hard and soft != resource.RLIM_INFINITY:
try:
limit = 8192 if hard == resource.RLIM_INFINITY else hard
resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
except OSError:
_log.exception('error setting open file limits')
else:
_log.debug('open file resource limit increased from %d to %d',
soft, limit)
# Set configuration
if HAVE_RESTRICTED:
if opts.verify_agents:
_log.info('Agent integrity verification enabled')
if opts.resource_monitor:
_log.info('Resource monitor enabled')
opts.resmon = resmon.ResourceMonitor()
opts.aip = aip.AIPplatform(opts)
opts.aip.setup()
# Check for secure mode/permissions on VOLTTRON_HOME directory
mode = os.stat(volttron_home).st_mode
if mode & (stat.S_IWGRP | stat.S_IWOTH):
_log.warning('insecure mode on directory: %s', volttron_home)
# Get or generate encryption key
keyfile = os.path.join(volttron_home, 'curve.key')
_log.debug('using key file %s', keyfile)
try:
st = os.stat(keyfile)
except OSError as exc:
if exc.errno != errno.ENOENT:
parser.error(str(exc))
# Key doesn't exist, so create it securely
_log.info('generating missing key file')
try:
fd = os.open(keyfile, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
except OSError as exc:
parser.error(str(exc))
try:
key = ''.join(curve_keypair())
os.write(fd, key)
finally:
os.close(fd)
else:
if st.st_mode & (stat.S_IRWXG | stat.S_IRWXO):
_log.warning('insecure mode on key file')
if not st.st_size:
_log.warning('empty key file; VIP encryption is disabled!')
key = ''
else:
# Allow two extra bytes in case someone opened the file with
# a text editor and it appended '\n' or '\r\n'.
if not 80 <= st.st_size <= 82:
_log.warning('key file is wrong size; connections may fail')
with open(keyfile) as infile:
key = infile.read(80)
publickey = key[:40]
if publickey:
_log.info('public key: %r (%s)', publickey, encode_key(publickey))
secretkey = key[40:]
# The following line doesn't appear to do anything, but it creates
# a context common to the green and non-green zmq modules.
zmq.Context.instance() # DO NOT REMOVE LINE!!
# Main loops
def router(stop):
try:
Router(opts.vip_local_address,
opts.vip_address,
secretkey=secretkey,
default_user_id=b'vip.service',
monitor=opts.monitor).run()
except Exception:
_log.exception('Unhandled exception in router loop')
finally:
stop()
address = 'inproc://vip'
try:
# Ensure auth service is running before router
auth = AuthService(address=address, identity='auth')
event = gevent.event.Event()
auth_task = gevent.spawn(auth.core.run, event)
event.wait()
del event
# Start router in separate thread to remain responsive
thread = threading.Thread(target=router, args=(auth.core.stop, ))
thread.daemon = True
thread.start()
# Launch additional services and wait for them to start before
# auto-starting agents
services = [
ControlService(opts.aip, address=address, identity='control'),
PubSubService(address=address, identity='pubsub'),
CompatPubSub(address=address,
identity='pubsub.compat',
publish_address=opts.publish_address,
subscribe_address=opts.subscribe_address),
]
events = [gevent.event.Event() for service in services]
tasks = [
gevent.spawn(service.core.run, event)
for service, event in zip(services, events)
]
tasks.append(auth_task)
gevent.wait(events)
del events
# Auto-start agents now that all services are up
if opts.autostart:
for name, error in opts.aip.autostart():
_log.error('error starting {!r}: {}\n'.format(name, error))
# Wait for any service to stop, signaling exit
try:
gevent.wait(tasks, count=1)
except KeyboardInterrupt:
_log.info('SIGINT received; shutting down')
sys.stderr.write('Shutting down.\n')
for task in tasks:
task.kill(block=False)
gevent.wait(tasks)
finally:
opts.aip.finish()
def start_volttron_process(opts):
'''Start the main volttron process.
Typically this function is used from main.py and just uses the argparser's
Options arguments as inputs. It also can be called with a dictionary. In
that case the dictionaries keys are mapped into a value that acts like the
args options.
'''
if isinstance(opts, dict):
opts = type('Options', (), opts)()
# vip_address is meant to be a list so make it so.
if not isinstance(opts.vip_address, list):
opts.vip_address = [opts.vip_address]
if opts.log:
opts.log = config.expandall(opts.log)
if opts.log_config:
opts.log_config = config.expandall(opts.log_config)
opts.publish_address = config.expandall(opts.publish_address)
opts.subscribe_address = config.expandall(opts.subscribe_address)
opts.vip_address = [config.expandall(addr) for addr in opts.vip_address]
opts.vip_local_address = config.expandall(opts.vip_local_address)
import urlparse
if opts.bind_web_address:
parsed = urlparse.urlparse(opts.bind_web_address)
if not parsed.scheme:
raise StandardError(
'bind-web-address must begin with http or https.')
opts.bind_web_address = config.expandall(opts.bind_web_address)
if opts.volttron_central_address:
parsed = urlparse.urlparse(opts.volttron_central_address)
if not parsed.scheme:
raise StandardError(
'volttron-central-address must begin with http or https.')
opts.volttron_central_address = config.expandall(
opts.volttron_central_address)
if getattr(opts, 'show_config', False):
for name, value in sorted(vars(opts).iteritems()):
print(name, repr(value))
return
# Configure logging
level = max(1, opts.verboseness)
if opts.monitor and level > logging.INFO:
level = logging.INFO
if opts.log is None:
log_to_file(sys.stderr, level)
elif opts.log == '-':
log_to_file(sys.stdout, level)
elif opts.log:
log_to_file(opts.log, level, handler_class=handlers.WatchedFileHandler)
else:
log_to_file(None, 100, handler_class=lambda x: logging.NullHandler())
if opts.log_config:
error = configure_logging(opts.log_config)
if error:
parser.error('{}: {}'.format(*error))
# Increase open files resource limit to max or 8192 if unlimited
try:
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
except OSError:
_log.exception('error getting open file limits')
else:
if soft != hard and soft != resource.RLIM_INFINITY:
try:
limit = 8192 if hard == resource.RLIM_INFINITY else hard
resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
except OSError:
_log.exception('error setting open file limits')
else:
_log.debug('open file resource limit increased from %d to %d',
soft, limit)
# Set configuration
if HAVE_RESTRICTED:
if opts.verify_agents:
_log.info('Agent integrity verification enabled')
if opts.resource_monitor:
_log.info('Resource monitor enabled')
opts.resmon = resmon.ResourceMonitor()
opts.aip = aip.AIPplatform(opts)
opts.aip.setup()
# Check for secure mode/permissions on VOLTTRON_HOME directory
mode = os.stat(opts.volttron_home).st_mode
if mode & (stat.S_IWGRP | stat.S_IWOTH):
_log.warning('insecure mode on directory: %s', opts.volttron_home)
# Get or generate encryption key
if opts.developer_mode:
secretkey = None
publickey = None
_log.warning('developer mode enabled; '
'authentication and encryption are disabled!')
else:
keyfile = os.path.join(opts.volttron_home, 'curve.key')
_log.debug('using key file %s', keyfile)
try:
st = os.stat(keyfile)
except OSError as exc:
if exc.errno != errno.ENOENT:
parser.error(str(exc))
# Key doesn't exist, so create it securely
_log.info('generating missing key file')
try:
fd = os.open(keyfile, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
except OSError as exc:
parser.error(str(exc))
try:
key = ''.join(curve_keypair())
os.write(fd, key)
finally:
os.close(fd)
else:
if st.st_mode & (stat.S_IRWXG | stat.S_IRWXO):
_log.warning('insecure mode on key file')
if not st.st_size:
_log.warning('empty key file; VIP encryption is disabled!')
key = ''
else:
# Allow two extra bytes in case someone opened the file with
# a text editor and it appended '\n' or '\r\n'.
if not 80 <= st.st_size <= 82:
_log.warning('key file is wrong size; connections may fail')
with open(keyfile) as infile:
key = infile.read(80)
publickey = key[:40]
if publickey:
_log.info('public key: %s', encode_key(publickey))
secretkey = key[40:]
# The following line doesn't appear to do anything, but it creates
# a context common to the green and non-green zmq modules.
zmq.Context.instance() # DO NOT REMOVE LINE!!
tracker = Tracker()
# Main loops
def router(stop):
try:
Router(opts.vip_local_address, opts.vip_address,
secretkey=secretkey, default_user_id=b'vip.service',
monitor=opts.monitor, tracker=tracker).run()
except Exception:
_log.exception('Unhandled exception in router loop')
raise
finally:
stop()
address = 'inproc://vip'
try:
# Ensure auth service is running before router
auth_file = os.path.join(opts.volttron_home, 'auth.json')
auth = AuthService(
auth_file, opts.aip, address=address, identity='auth',
allow_any=opts.developer_mode)
event = gevent.event.Event()
auth_task = gevent.spawn(auth.core.run, event)
event.wait()
del event
# Start router in separate thread to remain responsive
thread = threading.Thread(target=router, args=(auth.core.stop,))
thread.daemon = True
thread.start()
gevent.sleep(0.1)
if not thread.isAlive():
sys.exit()
protected_topics_file = os.path.join(opts.volttron_home, 'protected_topics.json')
_log.debug('protected topics file %s', protected_topics_file)
# Launch additional services and wait for them to start before
# auto-starting agents
services = [
ControlService(opts.aip, address=address, identity='control', tracker=tracker, heartbeat_autostart=True),
PubSubService(protected_topics_file, address=address, identity='pubsub', heartbeat_autostart=True),
CompatPubSub(address=address, identity='pubsub.compat',
publish_address=opts.publish_address,
subscribe_address=opts.subscribe_address),
MasterWebService(
serverkey=publickey, identity=MASTER_WEB,
address=address,
bind_web_address=opts.bind_web_address,
volttron_central_address=opts.volttron_central_address,
aip=opts.aip)
]
events = [gevent.event.Event() for service in services]
tasks = [gevent.spawn(service.core.run, event)
for service, event in zip(services, events)]
tasks.append(auth_task)
gevent.wait(events)
del events
# Auto-start agents now that all services are up
if opts.autostart:
for name, error in opts.aip.autostart():
_log.error('error starting {!r}: {}\n'.format(name, error))
# Wait for any service to stop, signaling exit
try:
gevent.wait(tasks, count=1)
except KeyboardInterrupt:
_log.info('SIGINT received; shutting down')
sys.stderr.write('Shutting down.\n')
for task in tasks:
task.kill(block=False)
gevent.wait(tasks)
finally:
opts.aip.finish()