def run(self):
try:
if self.options.daemon: tools.daemon()
# set log level
if (self.options.verbose):
logger.set_level(logger.LOG_VERBOSE)
# Load /etc/planetlab/plc_config
config = Config(self.options.config)
try:
other_pid = tools.pid_file()
if other_pid != None:
print """There might be another instance of the node manager running as pid %d.
If this is not the case, please remove the pid file %s. -- exiting""" % (other_pid, tools.PID_FILE)
return
except OSError, err:
print "Warning while writing PID file:", err
# load modules
self.loaded_modules = []
for module in self.modules:
try:
m = __import__(module)
logger.verbose("nodemanager: triggering %s.start"%m.__name__)
m.start()
self.loaded_modules.append(m)
except ImportError, err:
print "Warning while loading module %s:" % module, err
def train(env,
seed,
policy_fn,
reward_giver,
reward_guidance,
expert_dataset,
algo,
g_step,
d_step,
policy_entcoeff,
num_timesteps,
save_per_iter,
checkpoint_dir,
log_dir,
pretrained,
BC_max_iter,
loss_percent,
task_name=None):
pretrained_weight = None
if pretrained and (BC_max_iter > 0):
# Pretrain with behavior cloning
import behavior_clone
pretrained_weight = behavior_clone.learn(env,
policy_fn,
expert_dataset,
max_iters=BC_max_iter)
rank = MPI.COMM_WORLD.Get_rank()
if rank != 0:
logger.set_level(logger.DISABLED)
workerseed = seed + 10000 * MPI.COMM_WORLD.Get_rank()
set_global_seeds(workerseed)
env.seed(workerseed)
learn(env,
policy_fn,
reward_giver,
reward_guidance,
expert_dataset,
rank,
pretrained=pretrained,
pretrained_weight=pretrained_weight,
g_step=g_step,
d_step=d_step,
entcoeff=policy_entcoeff,
max_timesteps=num_timesteps,
ckpt_dir=checkpoint_dir,
log_dir=log_dir,
save_per_iter=save_per_iter,
timesteps_per_batch=1024,
max_kl=0.01,
cg_iters=10,
cg_damping=0.1,
gamma=0.995,
lam=0.97,
algo=algo,
vf_iters=5,
vf_stepsize=1e-3,
loss_percent=loss_percent,
task_name=task_name)
def mult(num1: int, num2: int, verbosity: int = 4) -> int:
"""Multiply one number by another if they both are integers."""
set_level(verbosity, logger)
if are_int(num1, num2, logger):
return num1 * num2
else:
raise TypeError(TYPE_ERROR_MESSAGE)
def configure_logging(self):
"""Configure the experiment"""
if self.comm is None or self.rank == 0:
log_path = self.get_log_path()
formats_strs = ['stdout', 'log', 'csv']
fmtstr = "configuring logger"
if self.comm is not None and self.rank == 0:
fmtstr += " [master]"
logger.info(fmtstr)
logger.configure(dir_=log_path, format_strs=formats_strs)
fmtstr = "logger configured"
if self.comm is not None and self.rank == 0:
fmtstr += " [master]"
logger.info(fmtstr)
logger.info(" directory: {}".format(log_path))
logger.info(" output formats: {}".format(formats_strs))
# In the same log folder, log args in yaml in yaml file
file_logger = FileLogger(uuid=self.uuid,
path=self.get_log_path(),
file_prefix=self.name_prefix)
file_logger.set_info('note', self.args.note)
file_logger.set_info('uuid', self.uuid)
file_logger.set_info('task', self.args.task)
file_logger.set_info('args', str(self.args))
fmtstr = "experiment configured"
if self.comm is not None:
fmtstr += " [{} MPI workers]".format(self.comm.Get_size())
logger.info(fmtstr)
else:
logger.info("configuring logger [worker #{}]".format(self.rank))
logger.configure(dir_=None, format_strs=None)
logger.set_level(logger.DISABLED)
def add(num1: int, num2: int, verbosity: int = 4) -> int:
"""Add two numbers if they both are integers."""
set_level(verbosity, logger)
if are_int(num1, num2, logger):
return num1 + num2
else:
raise TypeError(TYPE_ERROR_MESSAGE)
def sub(num1: int, num2: int, verbosity: int = 4) -> int:
"""Subtract one number from another if they both are integers."""
set_level(verbosity, logger)
if are_int(num1, num2, logger):
return num1 - num2
else:
raise TypeError(TYPE_ERROR_MESSAGE)
def configure_logger(log_dir):
logger.configure(log_dir, format_strs=['log'])
global tb
tb = logger.Logger(log_dir, [
logger.make_output_format('tensorboard', log_dir),
logger.make_output_format('csv', log_dir),
logger.make_output_format('stdout', log_dir)
])
global log
logger.set_level(60)
log = logger.log
def main(args):
U.make_session(num_cpu=1).__enter__()
set_global_seeds(args.seed)
from dp_env_v3 import DPEnv
env = DPEnv
task_name = get_task_short_name(args)
if rank == 0:
logger.configure(dir='log_gail/%s' % task_name)
if rank != 0:
logger.set_level(logger.DISABLED)
def policy_fn(name, ob_space, ac_space, reuse=False):
return MlpPolicy(name=name,
ob_space=ob_space,
ac_space=ac_space,
reuse=reuse,
hid_size=args.policy_hidden_size,
num_hid_layers=2)
import logging
import os.path as osp
import bench
env = bench.Monitor(
env,
logger.get_dir() and osp.join(logger.get_dir(), "monitor.json"))
env.seed(args.seed)
gym.logger.setLevel(logging.WARN)
task_name = get_task_name(args)
args.checkpoint_dir = osp.join(args.checkpoint_dir, task_name)
args.log_dir = osp.join(args.log_dir, task_name)
if args.task == 'train':
dataset = Mujoco_Dset(expert_path=args.expert_path,
traj_limitation=args.traj_limitation)
reward_giver = TransitionClassifier(env,
args.adversary_hidden_size,
entcoeff=args.adversary_entcoeff)
train(env, args.seed, policy_fn, reward_giver, dataset, args.algo,
args.g_step, args.d_step, args.policy_entcoeff,
args.num_timesteps, args.save_per_iter, args.checkpoint_dir,
args.log_dir, args.pretrained, args.BC_max_iter, task_name)
elif args.task == 'evaluate':
runner(env,
policy_fn,
args.load_model_path,
timesteps_per_batch=1024,
number_trajs=10,
stochastic_policy=args.stochastic_policy,
save=args.save_sample)
else:
raise NotImplementedError
env.close()
def main(args):
U.make_session(num_cpu=1).__enter__()
set_global_seeds(args.seed)
# from dp_env_v2 import DPEnv
from dp_env_v3 import DPEnv
# from dp_env_test import DPEnv
env = DPEnv()
# env = gym.make('Humanoid-v2')
task_name = get_task_short_name(args)
def policy_fn(name, ob_space, ac_space, reuse=False):
return MlpPolicy(name=name, ob_space=ob_space, ac_space=ac_space,
reuse=reuse, hid_size=args.policy_hidden_size, num_hid_layers=2)
if args.task == 'train':
import logging
import os.path as osp
import bench
if MPI is None or MPI.COMM_WORLD.Get_rank() == 0:
logger.configure(dir='log_tmp/%s'%task_name)
if MPI.COMM_WORLD.Get_rank() != 0:
logger.set_level(logger.DISABLED)
env = bench.Monitor(env, logger.get_dir() and
osp.join(logger.get_dir(), "monitor.json"))
env.seed(args.seed)
gym.logger.setLevel(logging.WARN)
task_name = get_task_short_name(args)
args.checkpoint_dir = osp.join(args.checkpoint_dir, task_name)
args.log_dir = osp.join(args.log_dir, task_name)
train(env,
args.seed,
policy_fn,
args.g_step,
args.policy_entcoeff,
args.pretrained_weight_path,
args.num_timesteps,
args.save_per_iter,
args.checkpoint_dir,
args.log_dir,
task_name)
elif args.task == 'evaluate':
runner(env,
policy_fn,
args.load_model_path,
timesteps_per_batch=1024,
number_trajs=100,
stochastic_policy=args.stochastic_policy,
save=args.save_sample)
else:
raise NotImplementedError
env.close()
def run(self):
# make sure to create /etc/planetlab/virt so others can read that
# used e.g. in vsys-scripts's sliceip
tools.get_node_virt()
try:
if self.options.daemon: tools.daemon()
# set log level
if (self.options.verbose):
logger.set_level(logger.LOG_VERBOSE)
# Load /etc/planetlab/plc_config
config = Config(self.options.config)
try:
other_pid = tools.pid_file()
if other_pid != None:
print """There might be another instance of the node manager running as pid %d.
If this is not the case, please remove the pid file %s. -- exiting""" % (
other_pid, tools.PID_FILE)
return
except OSError, err:
print "Warning while writing PID file:", err
# load modules
self.loaded_modules = []
for module in self.modules:
try:
m = __import__(module)
logger.verbose("nodemanager: triggering %s.start" %
m.__name__)
m.start()
self.loaded_modules.append(m)
except ImportError, err:
logger.log_exc(
"ERROR while loading module %s - skipping:" % module)
# if we fail to load any of these, it's really no need to go on any further
if module in NodeManager.core_modules:
logger.log("FATAL : failed to load core module %s" %
module)
except AttributeError, err:
# triggered when module doesn't have a 'start' method
logger.log_exc(
"ERROR while starting module %s - skipping:" % module)
# if we fail to load any of these, it's really no need to go on any further
if module in NodeManager.core_modules:
logger.log("FATAL : failed to start core module %s" %
module)
def run(self):
# make sure to create /etc/planetlab/virt so others can read that
# used e.g. in vsys-scripts's sliceip
tools.get_node_virt()
try:
if self.options.daemon: tools.daemon()
# set log level
if (self.options.verbose):
logger.set_level(logger.LOG_VERBOSE)
# Load /etc/planetlab/plc_config
config = Config(self.options.config)
try:
other_pid = tools.pid_file()
if other_pid != None:
print """There might be another instance of the node manager running as pid %d.
If this is not the case, please remove the pid file %s. -- exiting""" % (other_pid, tools.PID_FILE)
return
except OSError, err:
print "Warning while writing PID file:", err
# load modules
self.loaded_modules = []
for module in self.modules:
try:
m = __import__(module)
logger.verbose("nodemanager: triggering %s.start"%m.__name__)
m.start()
self.loaded_modules.append(m)
except ImportError, err:
logger.log_exc ("ERROR while loading module %s - skipping:" % module)
# if we fail to load any of these, it's really no need to go on any further
if module in NodeManager.core_modules:
logger.log("FATAL : failed to load core module %s"%module)
except AttributeError, err:
# triggered when module doesn't have a 'start' method
logger.log_exc ("ERROR while starting module %s - skipping:" % module)
# if we fail to load any of these, it's really no need to go on any further
if module in NodeManager.core_modules:
logger.log("FATAL : failed to start core module %s"%module)
def main():
print "main: START"
Debug ( "debug msg -- ceci est un test" )
Info ( "info msg -- ceci est un test" )
Warn ( "warn msg -- ceci est un test" )
Error ( "error msg -- ceci est un test" )
Critical ( "critical msg -- ceci est un test" )
print "========"
logger.set_level("debug")
Debug ( "debug msg -- ceci est un test" )
Info ( "info msg -- ceci est un test" )
Warn ( "warn msg -- ceci est un test" )
Error ( "error msg -- ceci est un test" )
Critical ( "critical msg -- ceci est un test" )
print "========"
logger.set_logfile(path.join(getcwd(), "output.log"))
Debug ( "debug msg -- ceci est un test" )
Info ( "info msg -- ceci est un test" )
Warn ( "warn msg -- ceci est un test" )
Error ( "error msg -- ceci est un test" )
Critical ( "critical msg -- ceci est un test" )
print "========"
logger.set_level("warning")
Debug ( "debug msg -- ceci est un test" )
Info ( "info msg -- ceci est un test" )
Warn ( "warn msg -- ceci est un test" )
Error ( "error msg -- ceci est un test" )
Critical ( "critical msg -- ceci est un test" )
print "main: END"
version="MQTTany {}".format(__version__),
help="show version and exit",
)
return parser.parse_args()
if __name__ == "__main__":
if sys.version_info[:2] < (3, 6):
log.info("Detected Python verison {}".format(".".join(
[str(i) for i in sys.version_info[:3]])))
log.error("MQTTany requires a minimum Python version of 3.6")
exit(126)
args = get_args()
if args.verbose > 1:
logger.set_level(logger.LogLevel.TRACE)
elif args.verbose > 0:
logger.set_level(logger.LogLevel.DEBUG)
import core
from common import SignalHook
signal = SignalHook()
mproc.current_process().name = "core"
poison_pill = False
log.info("MQTTany {version} starting".format(version=__version__))
if len(__version__.split("-")) > 1:
log.warn("")
def __init__(self):
'''
Main Onionr class. This is for the CLI program, and does not handle much of the logic.
In general, external programs and plugins should not use this class.
'''
try:
os.chdir(sys.path[0])
except FileNotFoundError:
pass
# Load global configuration data
data_exists = os.path.exists('data/')
if not data_exists:
os.mkdir('data/')
if os.path.exists('static-data/default_config.json'):
config.set_config(
json.loads(open('static-data/default_config.json').read())
) # this is the default config, it will be overwritten if a config file already exists. Else, it saves it
else:
# the default config file doesn't exist, try hardcoded config
config.set_config({
'devmode': True,
'log': {
'file': {
'output': True,
'path': 'data/output.log'
},
'console': {
'output': True,
'color': True
}
}
})
if not data_exists:
config.save()
config.reload() # this will read the configuration file into memory
settings = 0b000
if config.get('log', {'console': {'color': True}})['console']['color']:
settings = settings | logger.USE_ANSI
if config.get('log', {'console': {
'output': True
}})['console']['output']:
settings = settings | logger.OUTPUT_TO_CONSOLE
if config.get('log', {'file': {'output': True}})['file']['output']:
settings = settings | logger.OUTPUT_TO_FILE
logger.set_file(
config.get('log', {'file': {
'path': 'data/output.log'
}})['file']['path'])
logger.set_settings(settings)
if str(config.get('devmode', True)).lower() == 'true':
self._developmentMode = True
logger.set_level(logger.LEVEL_DEBUG)
else:
self._developmentMode = False
logger.set_level(logger.LEVEL_INFO)
self.onionrCore = core.Core()
self.onionrUtils = OnionrUtils(self.onionrCore)
# Handle commands
self.debug = False # Whole application debugging
if os.path.exists('data-encrypted.dat'):
while True:
print('Enter password to decrypt:')
password = getpass.getpass()
result = self.onionrCore.dataDirDecrypt(password)
if os.path.exists('data/'):
break
else:
logger.error('Failed to decrypt: ' + result[1],
timestamp=False)
else:
# If data folder does not exist
if not data_exists:
if not os.path.exists('data/blocks/'):
os.mkdir('data/blocks/')
# Copy default plugins into plugins folder
if not os.path.exists(plugins.get_plugins_folder()):
if os.path.exists('static-data/default-plugins/'):
names = [
f for f in os.listdir("static-data/default-plugins/")
if not os.path.isfile(f)
]
shutil.copytree('static-data/default-plugins/',
plugins.get_plugins_folder())
# Enable plugins
for name in names:
if not name in plugins.get_enabled_plugins():
plugins.enable(name, self)
for name in plugins.get_enabled_plugins():
if not os.path.exists(plugins.get_plugin_data_folder(name)):
try:
os.mkdir(plugins.get_plugin_data_folder(name))
except:
plugins.disable(name, onionr=self, stop_event=False)
if not os.path.exists(self.onionrCore.peerDB):
self.onionrCore.createPeerDB()
pass
if not os.path.exists(self.onionrCore.addressDB):
self.onionrCore.createAddressDB()
# Get configuration
if not data_exists:
# Generate default config
# Hostname should only be set if different from 127.x.x.x. Important for DNS rebinding attack prevention.
if self.debug:
randomPort = 8080
else:
while True:
randomPort = random.randint(1024, 65535)
if self.onionrUtils.checkPort(randomPort):
break
config.set(
'client', {
'participate': 'true',
'client_hmac': base64.b16encode(
os.urandom(32)).decode('utf-8'),
'port': randomPort,
'api_version': API_VERSION
}, True)
self.cmds = {
'': self.showHelpSuggestion,
'help': self.showHelp,
'version': self.version,
'config': self.configure,
'start': self.start,
'stop': self.killDaemon,
'status': self.showStats,
'statistics': self.showStats,
'stats': self.showStats,
'enable-plugin': self.enablePlugin,
'enplugin': self.enablePlugin,
'enableplugin': self.enablePlugin,
'enmod': self.enablePlugin,
'disable-plugin': self.disablePlugin,
'displugin': self.disablePlugin,
'disableplugin': self.disablePlugin,
'dismod': self.disablePlugin,
'reload-plugin': self.reloadPlugin,
'reloadplugin': self.reloadPlugin,
'reload-plugins': self.reloadPlugin,
'reloadplugins': self.reloadPlugin,
'create-plugin': self.createPlugin,
'createplugin': self.createPlugin,
'plugin-create': self.createPlugin,
'listkeys': self.listKeys,
'list-keys': self.listKeys,
'addmsg': self.addMessage,
'addmessage': self.addMessage,
'add-msg': self.addMessage,
'add-message': self.addMessage,
'pm': self.sendEncrypt,
'getpms': self.getPMs,
'get-pms': self.getPMs,
'addpeer': self.addPeer,
'add-peer': self.addPeer,
'add-address': self.addAddress,
'add-addr': self.addAddress,
'addaddr': self.addAddress,
'addaddress': self.addAddress,
'addfile': self.addFile,
'importblocks': self.onionrUtils.importNewBlocks,
'introduce': self.onionrCore.introduceNode,
'connect': self.addAddress
}
self.cmdhelp = {
'help': 'Displays this Onionr help menu',
'version': 'Displays the Onionr version',
'config': 'Configures something and adds it to the file',
'start': 'Starts the Onionr daemon',
'stop': 'Stops the Onionr daemon',
'stats': 'Displays node statistics',
'enable-plugin': 'Enables and starts a plugin',
'disable-plugin': 'Disables and stops a plugin',
'reload-plugin': 'Reloads a plugin',
'create-plugin': 'Creates directory structure for a plugin',
'add-peer': 'Adds a peer to database',
'list-peers': 'Displays a list of peers',
'add-msg': 'Broadcasts a message to the Onionr network',
'pm': 'Adds a private message to block',
'get-pms': 'Shows private messages sent to you',
'addfile': 'Create an Onionr block from a file',
'importblocks':
'import blocks from the disk (Onionr is transport-agnostic!)',
'introduce': 'Introduce your node to the public Onionr network',
}
# initialize plugins
events.event('init', onionr=self, threaded=False)
command = ''
try:
command = sys.argv[1].lower()
except IndexError:
command = ''
finally:
self.execute(command)
if not self._developmentMode:
encryptionPassword = self.onionrUtils.getPassword(
'Enter password to encrypt directory: ')
self.onionrCore.dataDirEncrypt(encryptionPassword)
shutil.rmtree('data/')
return
default='127.0.0.1')
parser.add_argument("--port",
help="ASF IPC port. Default: 1242",
default='1242')
parser.add_argument("--password", help="ASF IPC password.", default=None)
parser.add_argument("--token",
type=str,
help="Telegram API token given by @botfather.",
default=None)
parser.add_argument("--alias",
type=str,
help="Telegram alias of the bot owner.",
default=None)
args = parser.parse_args()
logger.set_level(args.verbosity)
# Telegram related environment variables.
try:
args.token = os.environ[_ENV_TELEGRAM_BOT_TOKEN]
except KeyError as key_error:
if not args.token:
LOG.critical(
"No telegram bot token provided. Please do so using --token argument or %s environment variable.",
_ENV_TELEGRAM_BOT_TOKEN)
exit(1)
try:
args.alias = os.environ[_ENV_TELEGRAM_USER_ALIAS]
except KeyError as key_error:
if not args.alias:
nargs='?',
default="/etc/mqttany/mqttany.yml",
metavar="CONFIG_FILE",
help="path to configuration file (default '/etc/mqttany/mqttany.yml')")
parser.add_argument("-V",
"--version",
action="version",
version="MQTTany {}".format(__version__),
help="show version and exit")
return parser.parse_args()
if __name__ == '__main__':
args = get_args()
if args.verbose > 0:
logger.set_level(logger.DEBUG)
mproc.current_process().name = "mqttany"
killer = GracefulKiller()
poison_pill = False
log.info("MQTTany {version} starting".format(version=__version__))
try:
if not modules.load(args.config_file): exit(1)
while not killer.kill_now:
try: # to get an item from the queue
message = queue.get_nowait()
except QueueEmptyError:
time.sleep(0.1) # 100ms
def __init__(self, debug):
'''
Initialize the api server, preping variables for later use
This initilization defines all of the API entry points and handlers for the endpoints and errors
This also saves the used host (random localhost IP address) to the data folder in host.txt
'''
config.reload()
if config.get('devmode', True):
self._developmentMode = True
logger.set_level(logger.LEVEL_DEBUG)
else:
self._developmentMode = False
logger.set_level(logger.LEVEL_INFO)
self.debug = debug
self._privateDelayTime = 3
self._core = Core()
self._crypto = onionrcrypto.OnionrCrypto(self._core)
self._utils = onionrutils.OnionrUtils(self._core)
app = flask.Flask(__name__)
bindPort = int(config.get('client')['port'])
self.bindPort = bindPort
self.clientToken = config.get('client')['client_hmac']
if not os.environ.get("WERKZEUG_RUN_MAIN") == "true":
logger.debug('Your HMAC token: ' + logger.colors.underline +
self.clientToken)
if not debug and not self._developmentMode:
hostNums = [
random.randint(1, 255),
random.randint(1, 255),
random.randint(1, 255)
]
self.host = '127.' + str(hostNums[0]) + '.' + str(
hostNums[1]) + '.' + str(hostNums[2])
else:
self.host = '127.0.0.1'
hostFile = open('data/host.txt', 'w')
hostFile.write(self.host)
hostFile.close()
@app.before_request
def beforeReq():
'''
Simply define the request as not having yet failed, before every request.
'''
self.requestFailed = False
return
@app.after_request
def afterReq(resp):
if not self.requestFailed:
resp.headers['Access-Control-Allow-Origin'] = '*'
else:
resp.headers['server'] = 'Onionr'
resp.headers['Content-Type'] = 'text/plain'
resp.headers["Content-Security-Policy"] = "default-src 'none'"
resp.headers['X-Frame-Options'] = 'deny'
resp.headers['X-Content-Type-Options'] = "nosniff"
return resp
@app.route('/client/')
def private_handler():
startTime = math.floor(time.time())
# we should keep a hash DB of requests (with hmac) to prevent replays
action = request.args.get('action')
#if not self.debug:
token = request.args.get('token')
if not self.validateToken(token):
abort(403)
self.validateHost('private')
if action == 'hello':
resp = Response('Hello, World! ' + request.host)
elif action == 'shutdown':
request.environ.get('werkzeug.server.shutdown')()
resp = Response('Goodbye')
elif action == 'stats':
resp = Response('me_irl')
else:
resp = Response('(O_o) Dude what? (invalid command)')
endTime = math.floor(time.time())
elapsed = endTime - startTime
if elapsed < self._privateDelayTime:
time.sleep(self._privateDelayTime - elapsed)
return resp
@app.route('/public/')
def public_handler():
# Public means it is publicly network accessible
self.validateHost('public')
action = request.args.get('action')
requestingPeer = request.args.get('myID')
data = request.args.get('data')
if action == 'firstConnect':
pass
elif action == 'ping':
resp = Response("pong!")
elif action == 'getHMAC':
resp = Response(self._crypto.generateSymmetric())
elif action == 'getSymmetric':
resp = Response(self._crypto.generateSymmetric())
elif action == 'getDBHash':
resp = Response(self._utils.getBlockDBHash())
elif action == 'getBlockHashes':
resp = Response(self._core.getBlockList())
# setData should be something the communicator initiates, not this api
elif action == 'getData':
resp = self._core.getData(data)
if resp == False:
abort(404)
resp = ""
resp = Response(resp)
elif action == 'pex':
response = ','.join(self._core.listAdders())
if len(response) == 0:
response = 'none'
resp = Response(response)
elif action == 'kex':
response = ','.join(self._core.listPeers())
if len(response) == 0:
response = 'none'
resp = Response(response)
else:
resp = Response("")
return resp
@app.errorhandler(404)
def notfound(err):
self.requestFailed = True
resp = Response("")
return resp
@app.errorhandler(403)
def authFail(err):
self.requestFailed = True
resp = Response("403")
return resp
@app.errorhandler(401)
def clientError(err):
self.requestFailed = True
resp = Response("Invalid request")
return resp
if not os.environ.get("WERKZEUG_RUN_MAIN") == "true":
logger.info('Starting client on ' + self.host + ':' +
str(bindPort) + '...')
try:
app.run(host=self.host, port=bindPort, debug=True, threaded=True)
except Exception as e:
logger.error(str(e))
logger.fatal('Failed to start client on ' + self.host + ':' +
str(bindPort) + ', exiting...')
exit(1)
def run(self):
# make sure to create /etc/planetlab/virt so others can read that
# used e.g. in vsys-scripts's sliceip
tools.get_node_virt()
try:
if self.options.daemon:
tools.daemon()
# set log level
if self.options.verbose:
logger.set_level(logger.LOG_VERBOSE)
tools.init_signals()
# Load /etc/planetlab/plc_config
config = Config(self.options.config)
try:
other_pid = tools.pid_file()
if other_pid is not None:
print("""There might be another instance of the node manager running as pid {}.
If this is not the case, please remove the pid file {}. -- exiting"""
.format(other_pid, tools.PID_FILE))
return
except OSError as err:
print("Warning while writing PID file:", err)
# load modules
self.loaded_modules = []
for module in self.modules:
try:
m = __import__(module)
logger.verbose("nodemanager: triggering {}.start".format(m.__name__))
try:
m.start()
except Exception:
logger.log("WARNING: module {} did not start".format(m.__name__))
self.loaded_modules.append(m)
except Exception:
if module not in NodeManager.core_modules:
logger.log_exc("ERROR while loading module {} - skipped"
.format(module))
else:
logger.log("FATAL : failed to start core module {}".format(module))
sys.exit(1)
# sort on priority (lower first)
def module_priority(module):
return getattr(module, 'priority', NodeManager.default_priority)
self.loaded_modules.sort(key=module_priority)
logger.log('ordered modules:')
for module in self.loaded_modules:
logger.log('{}: {}'
.format(getattr(module, 'priority',
NodeManager.default_priority),
module.__name__))
# Load /etc/planetlab/session
if os.path.exists(self.options.session):
with open(self.options.session) as feed:
session = feed.read().strip()
else:
session = None
# get random periods
iperiod = int(self.options.period)
irandom = int(self.options.random)
# Initialize XML-RPC client
plc = PLCAPI(config.plc_api_uri, config.cacert, session, timeout=iperiod/2)
#check auth
logger.log("nodemanager: Checking Auth.")
while not plc.check_authentication():
try:
plc.update_session()
logger.log("nodemanager: Authentication Failure. Retrying")
except Exception as exc:
logger.log("nodemanager: Retry Failed. ({}); Waiting.."
.format(exc))
time.sleep(iperiod)
logger.log("nodemanager: Authentication Succeeded!")
while True:
# Main nodemanager Loop
work_beg = time.time()
logger.log('nodemanager: mainloop - calling GetSlivers - period={} random={}'
.format(iperiod, irandom))
self.GetSlivers(config, plc)
delay = iperiod + random.randrange(0, irandom)
work_end = time.time()
work_duration = int(work_end-work_beg)
logger.log('nodemanager: mainloop has worked for {} s - sleeping for {} s'
.format(work_duration, delay))
time.sleep(delay)
except SystemExit:
pass
except:
logger.log_exc("nodemanager: failed in run")
def run(env_id, seed, noise_type, layer_norm, evaluation, **kwargs):
# Configure things.
rank = MPI.COMM_WORLD.Get_rank()
if rank != 0:
logger.set_level(logger.DISABLED)
# Create envs.
env = gym.make(env_id)
env = Monitor(env, logger.get_dir() and os.path.join(logger.get_dir(), str(rank)))
if evaluation and rank == 0:
eval_env = gym.make(env_id)
eval_env = Monitor(eval_env, os.path.join(logger.get_dir(), 'gym_eval'))
env = Monitor(env, None)
else:
eval_env = None
# Parse noise_type
action_noise = None
param_noise = None
nb_actions = env.action_space.shape[-1]
for current_noise_type in noise_type.split(','):
current_noise_type = current_noise_type.strip()
if current_noise_type == 'none':
pass
elif 'adaptive-param' in current_noise_type:
_, stddev = current_noise_type.split('_')
param_noise = AdaptiveParamNoiseSpec(initial_stddev=float(stddev), desired_action_stddev=float(stddev))
elif 'normal' in current_noise_type:
_, stddev = current_noise_type.split('_')
action_noise = NormalActionNoise(mu=np.zeros(nb_actions), sigma=float(stddev) * np.ones(nb_actions))
elif 'ou' in current_noise_type:
_, stddev = current_noise_type.split('_')
action_noise = OrnsteinUhlenbeckActionNoise(mu=np.zeros(nb_actions),
sigma=float(stddev) * np.ones(nb_actions))
else:
raise RuntimeError('unknown noise type "{}"'.format(current_noise_type))
# Configure components.
memory = Memory(limit=int(1e6), action_shape=env.action_space.shape, observation_shape=env.observation_space.shape)
critic = Critic(layer_norm=layer_norm)
actor = Actor(nb_actions, layer_norm=layer_norm)
# Seed everything to make things reproducible.
seed = seed + 1000000 * rank
logger.info('rank {}: seed={}, logdir={}'.format(rank, seed, logger.get_dir()))
tf.reset_default_graph()
set_global_seeds(seed)
env.seed(seed)
if eval_env is not None:
eval_env.seed(seed)
# Disable logging for rank != 0 to avoid noise.
if rank == 0:
start_time = time.time()
training.train(env=env, eval_env=eval_env, param_noise=param_noise,
action_noise=action_noise, actor=actor, critic=critic, memory=memory, **kwargs)
env.close()
if eval_env is not None:
eval_env.close()
if rank == 0:
logger.info('total runtime: {}s'.format(time.time() - start_time))
def __init__(self):
'''
Main Onionr class. This is for the CLI program, and does not handle much of the logic.
In general, external programs and plugins should not use this class.
'''
try:
os.chdir(sys.path[0])
except FileNotFoundError:
pass
# Load global configuration data
exists = os.path.exists(config.get_config_file())
config.set_config(
{
'devmode': True,
'log': {
'file': {
'output': True,
'path': 'data/output.log'
},
'console': {
'output': True,
'color': True
}
}
}
) # this is the default config, it will be overwritten if a config file already exists. Else, it saves it
config.reload() # this will read the configuration file into memory
settings = 0b000
if config.get('log', {'console': {'color': True}})['console']['color']:
settings = settings | logger.USE_ANSI
if config.get('log', {'console': {
'output': True
}})['console']['output']:
settings = settings | logger.OUTPUT_TO_CONSOLE
if config.get('log', {'file': {'output': True}})['file']['output']:
settings = settings | logger.OUTPUT_TO_FILE
logger.set_file(
config.get('log', {'file': {
'path': 'data/output.log'
}})['file']['path'])
logger.set_settings(settings)
if config.get('devmode', True):
self._developmentMode = True
logger.set_level(logger.LEVEL_DEBUG)
else:
self._developmentMode = False
logger.set_level(logger.LEVEL_INFO)
self.onionrCore = core.Core()
self.onionrUtils = OnionrUtils(self.onionrCore)
# Handle commands
self.debug = False # Whole application debugging
if os.path.exists('data-encrypted.dat'):
while True:
print('Enter password to decrypt:')
password = getpass.getpass()
result = self.onionrCore.dataDirDecrypt(password)
if os.path.exists('data/'):
break
else:
logger.error('Failed to decrypt: ' + result[1])
else:
if not os.path.exists('data/'):
os.mkdir('data/')
os.mkdir('data/blocks/')
if not os.path.exists(self.onionrCore.peerDB):
self.onionrCore.createPeerDB()
pass
if not os.path.exists(self.onionrCore.addressDB):
self.onionrCore.createAddressDB()
# Get configuration
if not exists:
# Generate default config
# Hostname should only be set if different from 127.x.x.x. Important for DNS rebinding attack prevention.
if self.debug:
randomPort = 8080
else:
while True:
randomPort = random.randint(1024, 65535)
if self.onionrUtils.checkPort(randomPort):
break
config.set(
'client', {
'participate': 'true',
'client_hmac': base64.b64encode(
os.urandom(32)).decode('utf-8'),
'port': randomPort,
'api_version': API_VERSION
}, True)
self.cmds = {
'': self.showHelpSuggestion,
'help': self.showHelp,
'version': self.version,
'config': self.configure,
'start': self.start,
'stop': self.killDaemon,
'stats': self.showStats,
'enable-plugin': self.enablePlugin,
'enplugin': self.enablePlugin,
'enableplugin': self.enablePlugin,
'enmod': self.enablePlugin,
'disable-plugin': self.disablePlugin,
'displugin': self.disablePlugin,
'disableplugin': self.disablePlugin,
'dismod': self.disablePlugin,
'reload-plugin': self.reloadPlugin,
'reloadplugin': self.reloadPlugin,
'reload-plugins': self.reloadPlugin,
'reloadplugins': self.reloadPlugin,
'listpeers': self.listPeers,
'list-peers': self.listPeers,
'addmsg': self.addMessage,
'addmessage': self.addMessage,
'add-msg': self.addMessage,
'add-message': self.addMessage,
'pm': self.sendEncrypt,
'gui': self.openGUI,
'addpeer': self.addPeer,
'add-peer': self.addPeer,
'add-address': self.addAddress,
'addaddress': self.addAddress,
'connect': self.addAddress
}
self.cmdhelp = {
'help': 'Displays this Onionr help menu',
'version': 'Displays the Onionr version',
'config': 'Configures something and adds it to the file',
'start': 'Starts the Onionr daemon',
'stop': 'Stops the Onionr daemon',
'stats': 'Displays node statistics',
'enable-plugin': 'Enables and starts a plugin',
'disable-plugin': 'Disables and stops a plugin',
'reload-plugin': 'Reloads a plugin',
'list-peers': 'Displays a list of peers',
'add-peer': 'Adds a peer (?)',
'add-msg': 'Broadcasts a message to the Onionr network',
'pm': 'Adds a private message to block',
'gui': 'Opens a graphical interface for Onionr'
}
command = ''
try:
command = sys.argv[1].lower()
except IndexError:
command = ''
finally:
self.execute(command)
if not self._developmentMode:
encryptionPassword = self.onionrUtils.getPassword(
'Enter password to encrypt directory: ')
self.onionrCore.dataDirEncrypt(encryptionPassword)
shutil.rmtree('data/')
return
def setup_config(dataDir, o_inst=None):
data_exists = os.path.exists(dataDir)
if not data_exists:
os.mkdir(dataDir)
config.reload()
if not os.path.exists(config._configfile):
if os.path.exists('static-data/default_config.json'):
# this is the default config, it will be overwritten if a config file already exists. Else, it saves it
with open('static-data/default_config.json', 'r') as configReadIn:
config.set_config(json.loads(configReadIn.read()))
else:
# the default config file doesn't exist, try hardcoded config
logger.warn(
'Default configuration file does not exist, switching to hardcoded fallback configuration!'
)
config.set_config({
'dev_mode': True,
'log': {
'file': {
'output': True,
'path': dataDir + 'output.log'
},
'console': {
'output': True,
'color': True
}
}
})
config.save()
settings = 0b000
if config.get('log.console.color', True):
settings = settings | logger.USE_ANSI
if config.get('log.console.output', True):
settings = settings | logger.OUTPUT_TO_CONSOLE
if config.get('log.file.output', True):
settings = settings | logger.OUTPUT_TO_FILE
logger.set_settings(settings)
if not o_inst is None:
if str(config.get('general.dev_mode', True)).lower() == 'true':
o_inst._developmentMode = True
logger.set_level(logger.LEVEL_DEBUG)
else:
o_inst._developmentMode = False
logger.set_level(logger.LEVEL_INFO)
verbosity = str(config.get('log.verbosity', 'default')).lower().strip()
if not verbosity in ['default', 'null', 'none', 'nil']:
map = {
str(logger.LEVEL_DEBUG): logger.LEVEL_DEBUG,
'verbose': logger.LEVEL_DEBUG,
'debug': logger.LEVEL_DEBUG,
str(logger.LEVEL_INFO): logger.LEVEL_INFO,
'info': logger.LEVEL_INFO,
'information': logger.LEVEL_INFO,
str(logger.LEVEL_WARN): logger.LEVEL_WARN,
'warn': logger.LEVEL_WARN,
'warning': logger.LEVEL_WARN,
'warnings': logger.LEVEL_WARN,
str(logger.LEVEL_ERROR): logger.LEVEL_ERROR,
'err': logger.LEVEL_ERROR,
'error': logger.LEVEL_ERROR,
'errors': logger.LEVEL_ERROR,
str(logger.LEVEL_FATAL): logger.LEVEL_FATAL,
'fatal': logger.LEVEL_FATAL,
str(logger.LEVEL_IMPORTANT): logger.LEVEL_IMPORTANT,
'silent': logger.LEVEL_IMPORTANT,
'quiet': logger.LEVEL_IMPORTANT,
'important': logger.LEVEL_IMPORTANT
}
if verbosity in map:
logger.set_level(map[verbosity])
else:
logger.warn(
'Verbosity level %s is not valid, using default verbosity.' %
verbosity)
return data_exists
def __init__(self, debug):
'''
Initialize the api server, preping variables for later use
This initilization defines all of the API entry points and handlers for the endpoints and errors
This also saves the used host (random localhost IP address) to the data folder in host.txt
'''
config.reload()
if config.get('devmode', True):
self._developmentMode = True
logger.set_level(logger.LEVEL_DEBUG)
else:
self._developmentMode = False
logger.set_level(logger.LEVEL_INFO)
self.debug = debug
self._privateDelayTime = 3
self._core = Core()
self._crypto = onionrcrypto.OnionrCrypto(self._core)
self._utils = onionrutils.OnionrUtils(self._core)
app = flask.Flask(__name__)
bindPort = int(config.get('client')['port'])
self.bindPort = bindPort
self.clientToken = config.get('client')['client_hmac']
self.timeBypassToken = base64.b16encode(os.urandom(32)).decode()
self.i2pEnabled = config.get('i2p')['host']
self.mimeType = 'text/plain'
with open('data/time-bypass.txt', 'w') as bypass:
bypass.write(self.timeBypassToken)
if not os.environ.get("WERKZEUG_RUN_MAIN") == "true":
logger.debug('Your web password (KEEP SECRET): ' +
logger.colors.underline + self.clientToken)
if not debug and not self._developmentMode:
hostNums = [
random.randint(1, 255),
random.randint(1, 255),
random.randint(1, 255)
]
self.host = '127.' + str(hostNums[0]) + '.' + str(
hostNums[1]) + '.' + str(hostNums[2])
else:
self.host = '127.0.0.1'
hostFile = open('data/host.txt', 'w')
hostFile.write(self.host)
hostFile.close()
@app.before_request
def beforeReq():
'''
Simply define the request as not having yet failed, before every request.
'''
self.requestFailed = False
return
@app.after_request
def afterReq(resp):
if not self.requestFailed:
resp.headers['Access-Control-Allow-Origin'] = '*'
#else:
# resp.headers['server'] = 'Onionr'
resp.headers['Content-Type'] = self.mimeType
resp.headers[
"Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'"
resp.headers['X-Frame-Options'] = 'deny'
resp.headers['X-Content-Type-Options'] = "nosniff"
resp.headers['server'] = 'Onionr'
# reset to text/plain to help prevent browser attacks
if self.mimeType != 'text/plain':
self.mimeType = 'text/plain'
return resp
@app.route('/client/')
def private_handler():
if request.args.get('timingToken') is None:
timingToken = ''
else:
timingToken = request.args.get('timingToken')
data = request.args.get('data')
try:
data = data
except:
data = ''
startTime = math.floor(time.time())
# we should keep a hash DB of requests (with hmac) to prevent replays
action = request.args.get('action')
#if not self.debug:
token = request.args.get('token')
if not self.validateToken(token):
abort(403)
self.validateHost('private')
if action == 'hello':
resp = Response('Hello, World! ' + request.host)
elif action == 'shutdown':
# request.environ.get('werkzeug.server.shutdown')()
self.http_server.stop()
resp = Response('Goodbye')
elif action == 'ping':
resp = Response('pong')
elif action == 'stats':
resp = Response('me_irl')
raise Exception
elif action == 'site':
block = data
siteData = self._core.getData(data)
response = 'not found'
if siteData != '' and siteData != False:
self.mimeType = 'text/html'
response = siteData.split(b'-', 2)[-1]
resp = Response(response)
else:
resp = Response('(O_o) Dude what? (invalid command)')
endTime = math.floor(time.time())
elapsed = endTime - startTime
# if bypass token not used, delay response to prevent timing attacks
if not hmac.compare_digest(timingToken, self.timeBypassToken):
if elapsed < self._privateDelayTime:
time.sleep(self._privateDelayTime - elapsed)
return resp
@app.route('/')
def banner():
self.mimeType = 'text/html'
self.validateHost('public')
try:
with open('static-data/index.html', 'r') as html:
resp = Response(html.read())
except FileNotFoundError:
resp = Response("")
return resp
@app.route('/public/')
def public_handler():
# Public means it is publicly network accessible
self.validateHost('public')
action = request.args.get('action')
requestingPeer = request.args.get('myID')
data = request.args.get('data')
try:
data = data
except:
data = ''
if action == 'firstConnect':
pass
elif action == 'ping':
resp = Response("pong!")
elif action == 'getHMAC':
resp = Response(self._crypto.generateSymmetric())
elif action == 'getSymmetric':
resp = Response(self._crypto.generateSymmetric())
elif action == 'getDBHash':
resp = Response(self._utils.getBlockDBHash())
elif action == 'getBlockHashes':
resp = Response('\n'.join(self._core.getBlockList()))
elif action == 'directMessage':
resp = Response(self._core.handle_direct_connection(data))
elif action == 'announce':
if data != '':
# TODO: require POW for this
if self._core.addAddress(data):
resp = Response('Success')
else:
resp = Response('')
else:
resp = Response('')
# setData should be something the communicator initiates, not this api
elif action == 'getData':
if self._utils.validateHash(data):
if not os.path.exists('data/blocks/' + data + '.db'):
try:
resp = base64.b64encode(self._core.getData(data))
except TypeError:
resp = ""
if resp == False:
abort(404)
resp = ""
resp = Response(resp)
elif action == 'pex':
response = ','.join(self._core.listAdders())
if len(response) == 0:
response = 'none'
resp = Response(response)
elif action == 'kex':
peers = self._core.listPeers(getPow=True)
response = ','.join(peers)
resp = Response(response)
else:
resp = Response("")
return resp
@app.errorhandler(404)
def notfound(err):
self.requestFailed = True
resp = Response("")
return resp
@app.errorhandler(403)
def authFail(err):
self.requestFailed = True
resp = Response("403")
return resp
@app.errorhandler(401)
def clientError(err):
self.requestFailed = True
resp = Response("Invalid request")
return resp
if not os.environ.get("WERKZEUG_RUN_MAIN") == "true":
logger.info('Starting client on ' + self.host + ':' +
str(bindPort) + '...',
timestamp=True)
try:
self.http_server = WSGIServer((self.host, bindPort), app)
self.http_server.serve_forever()
except KeyboardInterrupt:
pass
#app.run(host=self.host, port=bindPort, debug=False, threaded=True)
except Exception as e:
logger.error(str(e))
logger.fatal('Failed to start client on ' + self.host + ':' +
str(bindPort) + ', exiting...')
exit(1)
