Exemplo n.º 1
0
    def run(self):
        try:
            if self.options.daemon: tools.daemon()

            # set log level
            if (self.options.verbose):
                logger.set_level(logger.LOG_VERBOSE)

            # Load /etc/planetlab/plc_config
            config = Config(self.options.config)

            try:
                other_pid = tools.pid_file()
                if other_pid != None:
                    print """There might be another instance of the node manager running as pid %d.
If this is not the case, please remove the pid file %s. -- exiting""" % (other_pid, tools.PID_FILE)
                    return
            except OSError, err:
                print "Warning while writing PID file:", err

            # load modules
            self.loaded_modules = []
            for module in self.modules:
                try:
                    m = __import__(module)
                    logger.verbose("nodemanager: triggering %s.start"%m.__name__)
                    m.start()
                    self.loaded_modules.append(m)
                except ImportError, err:
                    print "Warning while loading module %s:" % module, err
Exemplo n.º 2
0
def train(env,
          seed,
          policy_fn,
          reward_giver,
          reward_guidance,
          expert_dataset,
          algo,
          g_step,
          d_step,
          policy_entcoeff,
          num_timesteps,
          save_per_iter,
          checkpoint_dir,
          log_dir,
          pretrained,
          BC_max_iter,
          loss_percent,
          task_name=None):

    pretrained_weight = None
    if pretrained and (BC_max_iter > 0):
        # Pretrain with behavior cloning
        import behavior_clone
        pretrained_weight = behavior_clone.learn(env,
                                                 policy_fn,
                                                 expert_dataset,
                                                 max_iters=BC_max_iter)

    rank = MPI.COMM_WORLD.Get_rank()
    if rank != 0:
        logger.set_level(logger.DISABLED)
    workerseed = seed + 10000 * MPI.COMM_WORLD.Get_rank()
    set_global_seeds(workerseed)
    env.seed(workerseed)
    learn(env,
          policy_fn,
          reward_giver,
          reward_guidance,
          expert_dataset,
          rank,
          pretrained=pretrained,
          pretrained_weight=pretrained_weight,
          g_step=g_step,
          d_step=d_step,
          entcoeff=policy_entcoeff,
          max_timesteps=num_timesteps,
          ckpt_dir=checkpoint_dir,
          log_dir=log_dir,
          save_per_iter=save_per_iter,
          timesteps_per_batch=1024,
          max_kl=0.01,
          cg_iters=10,
          cg_damping=0.1,
          gamma=0.995,
          lam=0.97,
          algo=algo,
          vf_iters=5,
          vf_stepsize=1e-3,
          loss_percent=loss_percent,
          task_name=task_name)
Exemplo n.º 3
0
Arquivo: mult.py Projeto: stmsy/logger
def mult(num1: int, num2: int, verbosity: int = 4) -> int:
    """Multiply one number by another if they both are integers."""
    set_level(verbosity, logger)
    if are_int(num1, num2, logger):
        return num1 * num2
    else:
        raise TypeError(TYPE_ERROR_MESSAGE)
Exemplo n.º 4
0
 def configure_logging(self):
     """Configure the experiment"""
     if self.comm is None or self.rank == 0:
         log_path = self.get_log_path()
         formats_strs = ['stdout', 'log', 'csv']
         fmtstr = "configuring logger"
         if self.comm is not None and self.rank == 0:
             fmtstr += " [master]"
         logger.info(fmtstr)
         logger.configure(dir_=log_path, format_strs=formats_strs)
         fmtstr = "logger configured"
         if self.comm is not None and self.rank == 0:
             fmtstr += " [master]"
         logger.info(fmtstr)
         logger.info("  directory: {}".format(log_path))
         logger.info("  output formats: {}".format(formats_strs))
         # In the same log folder, log args in yaml in yaml file
         file_logger = FileLogger(uuid=self.uuid,
                                  path=self.get_log_path(),
                                  file_prefix=self.name_prefix)
         file_logger.set_info('note', self.args.note)
         file_logger.set_info('uuid', self.uuid)
         file_logger.set_info('task', self.args.task)
         file_logger.set_info('args', str(self.args))
         fmtstr = "experiment configured"
         if self.comm is not None:
             fmtstr += " [{} MPI workers]".format(self.comm.Get_size())
         logger.info(fmtstr)
     else:
         logger.info("configuring logger [worker #{}]".format(self.rank))
         logger.configure(dir_=None, format_strs=None)
         logger.set_level(logger.DISABLED)
Exemplo n.º 5
0
Arquivo: add.py Projeto: stmsy/logger
def add(num1: int, num2: int, verbosity: int = 4) -> int:
    """Add two numbers if they both are integers."""
    set_level(verbosity, logger)
    if are_int(num1, num2, logger):
        return num1 + num2
    else:
        raise TypeError(TYPE_ERROR_MESSAGE)
Exemplo n.º 6
0
def sub(num1: int, num2: int, verbosity: int = 4) -> int:
    """Subtract one number from another if they both are integers."""
    set_level(verbosity, logger)
    if are_int(num1, num2, logger):
        return num1 - num2
    else:
        raise TypeError(TYPE_ERROR_MESSAGE)
Exemplo n.º 7
0
def configure_logger(log_dir):
    logger.configure(log_dir, format_strs=['log'])
    global tb
    tb = logger.Logger(log_dir, [
        logger.make_output_format('tensorboard', log_dir),
        logger.make_output_format('csv', log_dir),
        logger.make_output_format('stdout', log_dir)
    ])
    global log
    logger.set_level(60)
    log = logger.log
Exemplo n.º 8
0
def main(args):
    U.make_session(num_cpu=1).__enter__()
    set_global_seeds(args.seed)
    from dp_env_v3 import DPEnv
    env = DPEnv

    task_name = get_task_short_name(args)

    if rank == 0:
        logger.configure(dir='log_gail/%s' % task_name)
    if rank != 0:
        logger.set_level(logger.DISABLED)

    def policy_fn(name, ob_space, ac_space, reuse=False):
        return MlpPolicy(name=name,
                         ob_space=ob_space,
                         ac_space=ac_space,
                         reuse=reuse,
                         hid_size=args.policy_hidden_size,
                         num_hid_layers=2)

    import logging
    import os.path as osp
    import bench
    env = bench.Monitor(
        env,
        logger.get_dir() and osp.join(logger.get_dir(), "monitor.json"))
    env.seed(args.seed)
    gym.logger.setLevel(logging.WARN)
    task_name = get_task_name(args)
    args.checkpoint_dir = osp.join(args.checkpoint_dir, task_name)
    args.log_dir = osp.join(args.log_dir, task_name)

    if args.task == 'train':
        dataset = Mujoco_Dset(expert_path=args.expert_path,
                              traj_limitation=args.traj_limitation)
        reward_giver = TransitionClassifier(env,
                                            args.adversary_hidden_size,
                                            entcoeff=args.adversary_entcoeff)
        train(env, args.seed, policy_fn, reward_giver, dataset, args.algo,
              args.g_step, args.d_step, args.policy_entcoeff,
              args.num_timesteps, args.save_per_iter, args.checkpoint_dir,
              args.log_dir, args.pretrained, args.BC_max_iter, task_name)
    elif args.task == 'evaluate':
        runner(env,
               policy_fn,
               args.load_model_path,
               timesteps_per_batch=1024,
               number_trajs=10,
               stochastic_policy=args.stochastic_policy,
               save=args.save_sample)
    else:
        raise NotImplementedError
    env.close()
Exemplo n.º 9
0
def main(args):
    U.make_session(num_cpu=1).__enter__()
    set_global_seeds(args.seed)
    # from dp_env_v2 import DPEnv
    from dp_env_v3 import DPEnv
    # from dp_env_test import DPEnv
    env = DPEnv()
    # env = gym.make('Humanoid-v2')

    task_name = get_task_short_name(args)

    def policy_fn(name, ob_space, ac_space, reuse=False):
        return MlpPolicy(name=name, ob_space=ob_space, ac_space=ac_space,
                                    reuse=reuse, hid_size=args.policy_hidden_size, num_hid_layers=2)

    if args.task == 'train':
        import logging
        import os.path as osp
        import bench
        if MPI is None or MPI.COMM_WORLD.Get_rank() == 0:
            logger.configure(dir='log_tmp/%s'%task_name)
        if MPI.COMM_WORLD.Get_rank() != 0:
            logger.set_level(logger.DISABLED)
        env = bench.Monitor(env, logger.get_dir() and
                            osp.join(logger.get_dir(), "monitor.json"))
        env.seed(args.seed)
        gym.logger.setLevel(logging.WARN)
        task_name = get_task_short_name(args)
        args.checkpoint_dir = osp.join(args.checkpoint_dir, task_name)
        args.log_dir = osp.join(args.log_dir, task_name)

        train(env,
              args.seed,
              policy_fn,
              args.g_step,
              args.policy_entcoeff,
              args.pretrained_weight_path,
              args.num_timesteps,
              args.save_per_iter,
              args.checkpoint_dir,
              args.log_dir,
              task_name)
    elif args.task == 'evaluate':
        runner(env,
               policy_fn,
               args.load_model_path,
               timesteps_per_batch=1024,
               number_trajs=100,
               stochastic_policy=args.stochastic_policy,
               save=args.save_sample)
    else:
        raise NotImplementedError
    env.close()
Exemplo n.º 10
0
    def run(self):
        # make sure to create /etc/planetlab/virt so others can read that
        # used e.g. in vsys-scripts's sliceip
        tools.get_node_virt()
        try:
            if self.options.daemon: tools.daemon()

            # set log level
            if (self.options.verbose):
                logger.set_level(logger.LOG_VERBOSE)

            # Load /etc/planetlab/plc_config
            config = Config(self.options.config)

            try:
                other_pid = tools.pid_file()
                if other_pid != None:
                    print """There might be another instance of the node manager running as pid %d.
If this is not the case, please remove the pid file %s. -- exiting""" % (
                        other_pid, tools.PID_FILE)
                    return
            except OSError, err:
                print "Warning while writing PID file:", err

            # load modules
            self.loaded_modules = []
            for module in self.modules:
                try:
                    m = __import__(module)
                    logger.verbose("nodemanager: triggering %s.start" %
                                   m.__name__)
                    m.start()
                    self.loaded_modules.append(m)
                except ImportError, err:
                    logger.log_exc(
                        "ERROR while loading module %s - skipping:" % module)
                    # if we fail to load any of these, it's really no need to go on any further
                    if module in NodeManager.core_modules:
                        logger.log("FATAL : failed to load core module %s" %
                                   module)
                except AttributeError, err:
                    # triggered when module doesn't have a 'start' method
                    logger.log_exc(
                        "ERROR while starting module %s - skipping:" % module)
                    # if we fail to load any of these, it's really no need to go on any further
                    if module in NodeManager.core_modules:
                        logger.log("FATAL : failed to start core module %s" %
                                   module)
Exemplo n.º 11
0
    def run(self):
        # make sure to create /etc/planetlab/virt so others can read that
        # used e.g. in vsys-scripts's sliceip
        tools.get_node_virt()
        try:
            if self.options.daemon: tools.daemon()

            # set log level
            if (self.options.verbose):
                logger.set_level(logger.LOG_VERBOSE)

            # Load /etc/planetlab/plc_config
            config = Config(self.options.config)

            try:
                other_pid = tools.pid_file()
                if other_pid != None:
                    print """There might be another instance of the node manager running as pid %d.
If this is not the case, please remove the pid file %s. -- exiting""" % (other_pid, tools.PID_FILE)
                    return
            except OSError, err:
                print "Warning while writing PID file:", err

            # load modules
            self.loaded_modules = []
            for module in self.modules:
                try:
                    m = __import__(module)
                    logger.verbose("nodemanager: triggering %s.start"%m.__name__)
                    m.start()
                    self.loaded_modules.append(m)
                except ImportError, err:
                    logger.log_exc ("ERROR while loading module %s - skipping:" % module)
                    # if we fail to load any of these, it's really no need to go on any further
                    if module in NodeManager.core_modules:
                        logger.log("FATAL : failed to load core module %s"%module)
                except AttributeError, err:
                    # triggered when module doesn't have a 'start' method
                    logger.log_exc ("ERROR while starting module %s - skipping:" % module)
                    # if we fail to load any of these, it's really no need to go on any further
                    if module in NodeManager.core_modules:
                        logger.log("FATAL : failed to start core module %s"%module)
Exemplo n.º 12
0
def main():
    print "main: START"

    Debug    ( "debug msg    -- ceci est un test" )
    Info     ( "info msg     -- ceci est un test" )
    Warn     ( "warn msg     -- ceci est un test" )
    Error    ( "error msg    -- ceci est un test" )
    Critical ( "critical msg -- ceci est un test" )

    print "========"

    logger.set_level("debug")
    Debug    ( "debug msg    -- ceci est un test" )
    Info     ( "info msg     -- ceci est un test" )
    Warn     ( "warn msg     -- ceci est un test" )
    Error    ( "error msg    -- ceci est un test" )
    Critical ( "critical msg -- ceci est un test" )

    print "========"

    logger.set_logfile(path.join(getcwd(), "output.log"))
    Debug    ( "debug msg    -- ceci est un test" )
    Info     ( "info msg     -- ceci est un test" )
    Warn     ( "warn msg     -- ceci est un test" )
    Error    ( "error msg    -- ceci est un test" )
    Critical ( "critical msg -- ceci est un test" )

    print "========"

    logger.set_level("warning")
    Debug    ( "debug msg    -- ceci est un test" )
    Info     ( "info msg     -- ceci est un test" )
    Warn     ( "warn msg     -- ceci est un test" )
    Error    ( "error msg    -- ceci est un test" )
    Critical ( "critical msg -- ceci est un test" )
    print "main: END"
Exemplo n.º 13
0
        version="MQTTany {}".format(__version__),
        help="show version and exit",
    )
    return parser.parse_args()


if __name__ == "__main__":
    if sys.version_info[:2] < (3, 6):
        log.info("Detected Python verison {}".format(".".join(
            [str(i) for i in sys.version_info[:3]])))
        log.error("MQTTany requires a minimum Python version of 3.6")
        exit(126)

    args = get_args()
    if args.verbose > 1:
        logger.set_level(logger.LogLevel.TRACE)
    elif args.verbose > 0:
        logger.set_level(logger.LogLevel.DEBUG)

    import core
    from common import SignalHook

    signal = SignalHook()

    mproc.current_process().name = "core"
    poison_pill = False

    log.info("MQTTany {version} starting".format(version=__version__))

    if len(__version__.split("-")) > 1:
        log.warn("")
Exemplo n.º 14
0
    def __init__(self):
        '''
            Main Onionr class. This is for the CLI program, and does not handle much of the logic.
            In general, external programs and plugins should not use this class.
        '''

        try:
            os.chdir(sys.path[0])
        except FileNotFoundError:
            pass

        # Load global configuration data

        data_exists = os.path.exists('data/')

        if not data_exists:
            os.mkdir('data/')

        if os.path.exists('static-data/default_config.json'):
            config.set_config(
                json.loads(open('static-data/default_config.json').read())
            )  # this is the default config, it will be overwritten if a config file already exists. Else, it saves it
        else:
            # the default config file doesn't exist, try hardcoded config
            config.set_config({
                'devmode': True,
                'log': {
                    'file': {
                        'output': True,
                        'path': 'data/output.log'
                    },
                    'console': {
                        'output': True,
                        'color': True
                    }
                }
            })
        if not data_exists:
            config.save()
        config.reload()  # this will read the configuration file into memory

        settings = 0b000
        if config.get('log', {'console': {'color': True}})['console']['color']:
            settings = settings | logger.USE_ANSI
        if config.get('log', {'console': {
                'output': True
        }})['console']['output']:
            settings = settings | logger.OUTPUT_TO_CONSOLE
        if config.get('log', {'file': {'output': True}})['file']['output']:
            settings = settings | logger.OUTPUT_TO_FILE
            logger.set_file(
                config.get('log', {'file': {
                    'path': 'data/output.log'
                }})['file']['path'])
        logger.set_settings(settings)

        if str(config.get('devmode', True)).lower() == 'true':
            self._developmentMode = True
            logger.set_level(logger.LEVEL_DEBUG)
        else:
            self._developmentMode = False
            logger.set_level(logger.LEVEL_INFO)

        self.onionrCore = core.Core()
        self.onionrUtils = OnionrUtils(self.onionrCore)

        # Handle commands

        self.debug = False  # Whole application debugging

        if os.path.exists('data-encrypted.dat'):
            while True:
                print('Enter password to decrypt:')
                password = getpass.getpass()
                result = self.onionrCore.dataDirDecrypt(password)
                if os.path.exists('data/'):
                    break
                else:
                    logger.error('Failed to decrypt: ' + result[1],
                                 timestamp=False)
        else:
            # If data folder does not exist
            if not data_exists:
                if not os.path.exists('data/blocks/'):
                    os.mkdir('data/blocks/')

            # Copy default plugins into plugins folder
            if not os.path.exists(plugins.get_plugins_folder()):
                if os.path.exists('static-data/default-plugins/'):
                    names = [
                        f for f in os.listdir("static-data/default-plugins/")
                        if not os.path.isfile(f)
                    ]
                    shutil.copytree('static-data/default-plugins/',
                                    plugins.get_plugins_folder())

                    # Enable plugins
                    for name in names:
                        if not name in plugins.get_enabled_plugins():
                            plugins.enable(name, self)

        for name in plugins.get_enabled_plugins():
            if not os.path.exists(plugins.get_plugin_data_folder(name)):
                try:
                    os.mkdir(plugins.get_plugin_data_folder(name))
                except:
                    plugins.disable(name, onionr=self, stop_event=False)

        if not os.path.exists(self.onionrCore.peerDB):
            self.onionrCore.createPeerDB()
            pass
        if not os.path.exists(self.onionrCore.addressDB):
            self.onionrCore.createAddressDB()

        # Get configuration

        if not data_exists:
            # Generate default config
            # Hostname should only be set if different from 127.x.x.x. Important for DNS rebinding attack prevention.
            if self.debug:
                randomPort = 8080
            else:
                while True:
                    randomPort = random.randint(1024, 65535)
                    if self.onionrUtils.checkPort(randomPort):
                        break
            config.set(
                'client', {
                    'participate': 'true',
                    'client_hmac': base64.b16encode(
                        os.urandom(32)).decode('utf-8'),
                    'port': randomPort,
                    'api_version': API_VERSION
                }, True)

        self.cmds = {
            '': self.showHelpSuggestion,
            'help': self.showHelp,
            'version': self.version,
            'config': self.configure,
            'start': self.start,
            'stop': self.killDaemon,
            'status': self.showStats,
            'statistics': self.showStats,
            'stats': self.showStats,
            'enable-plugin': self.enablePlugin,
            'enplugin': self.enablePlugin,
            'enableplugin': self.enablePlugin,
            'enmod': self.enablePlugin,
            'disable-plugin': self.disablePlugin,
            'displugin': self.disablePlugin,
            'disableplugin': self.disablePlugin,
            'dismod': self.disablePlugin,
            'reload-plugin': self.reloadPlugin,
            'reloadplugin': self.reloadPlugin,
            'reload-plugins': self.reloadPlugin,
            'reloadplugins': self.reloadPlugin,
            'create-plugin': self.createPlugin,
            'createplugin': self.createPlugin,
            'plugin-create': self.createPlugin,
            'listkeys': self.listKeys,
            'list-keys': self.listKeys,
            'addmsg': self.addMessage,
            'addmessage': self.addMessage,
            'add-msg': self.addMessage,
            'add-message': self.addMessage,
            'pm': self.sendEncrypt,
            'getpms': self.getPMs,
            'get-pms': self.getPMs,
            'addpeer': self.addPeer,
            'add-peer': self.addPeer,
            'add-address': self.addAddress,
            'add-addr': self.addAddress,
            'addaddr': self.addAddress,
            'addaddress': self.addAddress,
            'addfile': self.addFile,
            'importblocks': self.onionrUtils.importNewBlocks,
            'introduce': self.onionrCore.introduceNode,
            'connect': self.addAddress
        }

        self.cmdhelp = {
            'help': 'Displays this Onionr help menu',
            'version': 'Displays the Onionr version',
            'config': 'Configures something and adds it to the file',
            'start': 'Starts the Onionr daemon',
            'stop': 'Stops the Onionr daemon',
            'stats': 'Displays node statistics',
            'enable-plugin': 'Enables and starts a plugin',
            'disable-plugin': 'Disables and stops a plugin',
            'reload-plugin': 'Reloads a plugin',
            'create-plugin': 'Creates directory structure for a plugin',
            'add-peer': 'Adds a peer to database',
            'list-peers': 'Displays a list of peers',
            'add-msg': 'Broadcasts a message to the Onionr network',
            'pm': 'Adds a private message to block',
            'get-pms': 'Shows private messages sent to you',
            'addfile': 'Create an Onionr block from a file',
            'importblocks':
            'import blocks from the disk (Onionr is transport-agnostic!)',
            'introduce': 'Introduce your node to the public Onionr network',
        }

        # initialize plugins
        events.event('init', onionr=self, threaded=False)

        command = ''
        try:
            command = sys.argv[1].lower()
        except IndexError:
            command = ''
        finally:
            self.execute(command)

        if not self._developmentMode:
            encryptionPassword = self.onionrUtils.getPassword(
                'Enter password to encrypt directory: ')
            self.onionrCore.dataDirEncrypt(encryptionPassword)
            shutil.rmtree('data/')

        return
Exemplo n.º 15
0
                    default='127.0.0.1')
parser.add_argument("--port",
                    help="ASF IPC port. Default: 1242",
                    default='1242')
parser.add_argument("--password", help="ASF IPC password.", default=None)
parser.add_argument("--token",
                    type=str,
                    help="Telegram API token given by @botfather.",
                    default=None)
parser.add_argument("--alias",
                    type=str,
                    help="Telegram alias of the bot owner.",
                    default=None)
args = parser.parse_args()

logger.set_level(args.verbosity)

# Telegram related environment variables.
try:
    args.token = os.environ[_ENV_TELEGRAM_BOT_TOKEN]
except KeyError as key_error:
    if not args.token:
        LOG.critical(
            "No telegram bot token provided. Please do so using --token argument or %s environment variable.",
            _ENV_TELEGRAM_BOT_TOKEN)
        exit(1)

try:
    args.alias = os.environ[_ENV_TELEGRAM_USER_ALIAS]
except KeyError as key_error:
    if not args.alias:
Exemplo n.º 16
0
        nargs='?',
        default="/etc/mqttany/mqttany.yml",
        metavar="CONFIG_FILE",
        help="path to configuration file (default '/etc/mqttany/mqttany.yml')")
    parser.add_argument("-V",
                        "--version",
                        action="version",
                        version="MQTTany {}".format(__version__),
                        help="show version and exit")
    return parser.parse_args()


if __name__ == '__main__':
    args = get_args()
    if args.verbose > 0:
        logger.set_level(logger.DEBUG)

    mproc.current_process().name = "mqttany"
    killer = GracefulKiller()
    poison_pill = False

    log.info("MQTTany {version} starting".format(version=__version__))

    try:
        if not modules.load(args.config_file): exit(1)

        while not killer.kill_now:
            try:  # to get an item from the queue
                message = queue.get_nowait()
            except QueueEmptyError:
                time.sleep(0.1)  # 100ms
Exemplo n.º 17
0
    def __init__(self, debug):
        '''
            Initialize the api server, preping variables for later use

            This initilization defines all of the API entry points and handlers for the endpoints and errors
            This also saves the used host (random localhost IP address) to the data folder in host.txt
        '''

        config.reload()

        if config.get('devmode', True):
            self._developmentMode = True
            logger.set_level(logger.LEVEL_DEBUG)
        else:
            self._developmentMode = False
            logger.set_level(logger.LEVEL_INFO)

        self.debug = debug
        self._privateDelayTime = 3
        self._core = Core()
        self._crypto = onionrcrypto.OnionrCrypto(self._core)
        self._utils = onionrutils.OnionrUtils(self._core)
        app = flask.Flask(__name__)
        bindPort = int(config.get('client')['port'])
        self.bindPort = bindPort
        self.clientToken = config.get('client')['client_hmac']
        if not os.environ.get("WERKZEUG_RUN_MAIN") == "true":
            logger.debug('Your HMAC token: ' + logger.colors.underline +
                         self.clientToken)

        if not debug and not self._developmentMode:
            hostNums = [
                random.randint(1, 255),
                random.randint(1, 255),
                random.randint(1, 255)
            ]
            self.host = '127.' + str(hostNums[0]) + '.' + str(
                hostNums[1]) + '.' + str(hostNums[2])
        else:
            self.host = '127.0.0.1'
        hostFile = open('data/host.txt', 'w')
        hostFile.write(self.host)
        hostFile.close()

        @app.before_request
        def beforeReq():
            '''
                Simply define the request as not having yet failed, before every request.
            '''
            self.requestFailed = False

            return

        @app.after_request
        def afterReq(resp):
            if not self.requestFailed:
                resp.headers['Access-Control-Allow-Origin'] = '*'
            else:
                resp.headers['server'] = 'Onionr'
            resp.headers['Content-Type'] = 'text/plain'
            resp.headers["Content-Security-Policy"] = "default-src 'none'"
            resp.headers['X-Frame-Options'] = 'deny'
            resp.headers['X-Content-Type-Options'] = "nosniff"

            return resp

        @app.route('/client/')
        def private_handler():
            startTime = math.floor(time.time())
            # we should keep a hash DB of requests (with hmac) to prevent replays
            action = request.args.get('action')
            #if not self.debug:
            token = request.args.get('token')
            if not self.validateToken(token):
                abort(403)
            self.validateHost('private')
            if action == 'hello':
                resp = Response('Hello, World! ' + request.host)
            elif action == 'shutdown':
                request.environ.get('werkzeug.server.shutdown')()
                resp = Response('Goodbye')
            elif action == 'stats':
                resp = Response('me_irl')
            else:
                resp = Response('(O_o) Dude what? (invalid command)')
            endTime = math.floor(time.time())
            elapsed = endTime - startTime
            if elapsed < self._privateDelayTime:
                time.sleep(self._privateDelayTime - elapsed)

            return resp

        @app.route('/public/')
        def public_handler():
            # Public means it is publicly network accessible
            self.validateHost('public')
            action = request.args.get('action')
            requestingPeer = request.args.get('myID')
            data = request.args.get('data')
            if action == 'firstConnect':
                pass
            elif action == 'ping':
                resp = Response("pong!")
            elif action == 'getHMAC':
                resp = Response(self._crypto.generateSymmetric())
            elif action == 'getSymmetric':
                resp = Response(self._crypto.generateSymmetric())
            elif action == 'getDBHash':
                resp = Response(self._utils.getBlockDBHash())
            elif action == 'getBlockHashes':
                resp = Response(self._core.getBlockList())
            # setData should be something the communicator initiates, not this api
            elif action == 'getData':
                resp = self._core.getData(data)
                if resp == False:
                    abort(404)
                    resp = ""
                resp = Response(resp)
            elif action == 'pex':
                response = ','.join(self._core.listAdders())
                if len(response) == 0:
                    response = 'none'
                resp = Response(response)
            elif action == 'kex':
                response = ','.join(self._core.listPeers())
                if len(response) == 0:
                    response = 'none'
                resp = Response(response)
            else:
                resp = Response("")

            return resp

        @app.errorhandler(404)
        def notfound(err):
            self.requestFailed = True
            resp = Response("")

            return resp

        @app.errorhandler(403)
        def authFail(err):
            self.requestFailed = True
            resp = Response("403")

            return resp

        @app.errorhandler(401)
        def clientError(err):
            self.requestFailed = True
            resp = Response("Invalid request")

            return resp

        if not os.environ.get("WERKZEUG_RUN_MAIN") == "true":
            logger.info('Starting client on ' + self.host + ':' +
                        str(bindPort) + '...')

        try:
            app.run(host=self.host, port=bindPort, debug=True, threaded=True)
        except Exception as e:
            logger.error(str(e))
            logger.fatal('Failed to start client on ' + self.host + ':' +
                         str(bindPort) + ', exiting...')
            exit(1)
Exemplo n.º 18
0
    def run(self):
        # make sure to create /etc/planetlab/virt so others can read that
        # used e.g. in vsys-scripts's sliceip
        tools.get_node_virt()
        try:
            if self.options.daemon:
                tools.daemon()

            # set log level
            if self.options.verbose:
                logger.set_level(logger.LOG_VERBOSE)
            tools.init_signals()

            # Load /etc/planetlab/plc_config
            config = Config(self.options.config)

            try:
                other_pid = tools.pid_file()
                if other_pid is not None:
                    print("""There might be another instance of the node manager running as pid {}.
If this is not the case, please remove the pid file {}. -- exiting"""
                          .format(other_pid, tools.PID_FILE))
                    return
            except OSError as err:
                print("Warning while writing PID file:", err)

            # load modules
            self.loaded_modules = []
            for module in self.modules:
                try:
                    m = __import__(module)
                    logger.verbose("nodemanager: triggering {}.start".format(m.__name__))
                    try:
                        m.start()
                    except Exception:
                        logger.log("WARNING: module {} did not start".format(m.__name__))
                    self.loaded_modules.append(m)
                except Exception:
                    if module not in NodeManager.core_modules:
                        logger.log_exc("ERROR while loading module {} - skipped"
                                       .format(module))
                    else:
                        logger.log("FATAL : failed to start core module {}".format(module))
                        sys.exit(1)

            # sort on priority (lower first)
            def module_priority(module):
                return getattr(module, 'priority', NodeManager.default_priority)
            self.loaded_modules.sort(key=module_priority)

            logger.log('ordered modules:')
            for module in self.loaded_modules:
                logger.log('{}: {}'
                           .format(getattr(module, 'priority',
                                           NodeManager.default_priority),
                                   module.__name__))

            # Load /etc/planetlab/session
            if os.path.exists(self.options.session):
                with open(self.options.session) as feed:
                    session = feed.read().strip()
            else:
                session = None


            # get random periods
            iperiod = int(self.options.period)
            irandom = int(self.options.random)

            # Initialize XML-RPC client
            plc = PLCAPI(config.plc_api_uri, config.cacert, session, timeout=iperiod/2)

            #check auth
            logger.log("nodemanager: Checking Auth.")
            while not plc.check_authentication():
                try:
                    plc.update_session()
                    logger.log("nodemanager: Authentication Failure. Retrying")
                except Exception as exc:
                    logger.log("nodemanager: Retry Failed. ({}); Waiting.."
                               .format(exc))
                time.sleep(iperiod)
            logger.log("nodemanager: Authentication Succeeded!")


            while True:
            # Main nodemanager Loop
                work_beg = time.time()
                logger.log('nodemanager: mainloop - calling GetSlivers - period={} random={}'
                           .format(iperiod, irandom))
                self.GetSlivers(config, plc)
                delay = iperiod + random.randrange(0, irandom)
                work_end = time.time()
                work_duration = int(work_end-work_beg)
                logger.log('nodemanager: mainloop has worked for {} s - sleeping for {} s'
                           .format(work_duration, delay))
                time.sleep(delay)
        except SystemExit:
            pass
        except:
            logger.log_exc("nodemanager: failed in run")
Exemplo n.º 19
0
def run(env_id, seed, noise_type, layer_norm, evaluation, **kwargs):
    # Configure things.
    rank = MPI.COMM_WORLD.Get_rank()
    if rank != 0:
        logger.set_level(logger.DISABLED)

    # Create envs.
    env = gym.make(env_id)
    env = Monitor(env, logger.get_dir() and os.path.join(logger.get_dir(), str(rank)))

    if evaluation and rank == 0:
        eval_env = gym.make(env_id)
        eval_env = Monitor(eval_env, os.path.join(logger.get_dir(), 'gym_eval'))
        env = Monitor(env, None)
    else:
        eval_env = None

    # Parse noise_type
    action_noise = None
    param_noise = None
    nb_actions = env.action_space.shape[-1]
    for current_noise_type in noise_type.split(','):
        current_noise_type = current_noise_type.strip()
        if current_noise_type == 'none':
            pass
        elif 'adaptive-param' in current_noise_type:
            _, stddev = current_noise_type.split('_')
            param_noise = AdaptiveParamNoiseSpec(initial_stddev=float(stddev), desired_action_stddev=float(stddev))
        elif 'normal' in current_noise_type:
            _, stddev = current_noise_type.split('_')
            action_noise = NormalActionNoise(mu=np.zeros(nb_actions), sigma=float(stddev) * np.ones(nb_actions))
        elif 'ou' in current_noise_type:
            _, stddev = current_noise_type.split('_')
            action_noise = OrnsteinUhlenbeckActionNoise(mu=np.zeros(nb_actions),
                                                        sigma=float(stddev) * np.ones(nb_actions))
        else:
            raise RuntimeError('unknown noise type "{}"'.format(current_noise_type))

    # Configure components.
    memory = Memory(limit=int(1e6), action_shape=env.action_space.shape, observation_shape=env.observation_space.shape)
    critic = Critic(layer_norm=layer_norm)
    actor = Actor(nb_actions, layer_norm=layer_norm)

    # Seed everything to make things reproducible.
    seed = seed + 1000000 * rank
    logger.info('rank {}: seed={}, logdir={}'.format(rank, seed, logger.get_dir()))
    tf.reset_default_graph()
    set_global_seeds(seed)
    env.seed(seed)
    if eval_env is not None:
        eval_env.seed(seed)

    # Disable logging for rank != 0 to avoid noise.
    if rank == 0:
        start_time = time.time()
    training.train(env=env, eval_env=eval_env, param_noise=param_noise,
                   action_noise=action_noise, actor=actor, critic=critic, memory=memory, **kwargs)
    env.close()
    if eval_env is not None:
        eval_env.close()
    if rank == 0:
        logger.info('total runtime: {}s'.format(time.time() - start_time))
Exemplo n.º 20
0
    def __init__(self):
        '''
            Main Onionr class. This is for the CLI program, and does not handle much of the logic.
            In general, external programs and plugins should not use this class.
        '''

        try:
            os.chdir(sys.path[0])
        except FileNotFoundError:
            pass

        # Load global configuration data

        exists = os.path.exists(config.get_config_file())
        config.set_config(
            {
                'devmode': True,
                'log': {
                    'file': {
                        'output': True,
                        'path': 'data/output.log'
                    },
                    'console': {
                        'output': True,
                        'color': True
                    }
                }
            }
        )  # this is the default config, it will be overwritten if a config file already exists. Else, it saves it
        config.reload()  # this will read the configuration file into memory

        settings = 0b000
        if config.get('log', {'console': {'color': True}})['console']['color']:
            settings = settings | logger.USE_ANSI
        if config.get('log', {'console': {
                'output': True
        }})['console']['output']:
            settings = settings | logger.OUTPUT_TO_CONSOLE
        if config.get('log', {'file': {'output': True}})['file']['output']:
            settings = settings | logger.OUTPUT_TO_FILE
            logger.set_file(
                config.get('log', {'file': {
                    'path': 'data/output.log'
                }})['file']['path'])
        logger.set_settings(settings)

        if config.get('devmode', True):
            self._developmentMode = True
            logger.set_level(logger.LEVEL_DEBUG)
        else:
            self._developmentMode = False
            logger.set_level(logger.LEVEL_INFO)

        self.onionrCore = core.Core()
        self.onionrUtils = OnionrUtils(self.onionrCore)

        # Handle commands

        self.debug = False  # Whole application debugging

        if os.path.exists('data-encrypted.dat'):
            while True:
                print('Enter password to decrypt:')
                password = getpass.getpass()
                result = self.onionrCore.dataDirDecrypt(password)
                if os.path.exists('data/'):
                    break
                else:
                    logger.error('Failed to decrypt: ' + result[1])
        else:
            if not os.path.exists('data/'):
                os.mkdir('data/')
                os.mkdir('data/blocks/')

        if not os.path.exists(self.onionrCore.peerDB):
            self.onionrCore.createPeerDB()
            pass
        if not os.path.exists(self.onionrCore.addressDB):
            self.onionrCore.createAddressDB()

        # Get configuration

        if not exists:
            # Generate default config
            # Hostname should only be set if different from 127.x.x.x. Important for DNS rebinding attack prevention.
            if self.debug:
                randomPort = 8080
            else:
                while True:
                    randomPort = random.randint(1024, 65535)
                    if self.onionrUtils.checkPort(randomPort):
                        break
            config.set(
                'client', {
                    'participate': 'true',
                    'client_hmac': base64.b64encode(
                        os.urandom(32)).decode('utf-8'),
                    'port': randomPort,
                    'api_version': API_VERSION
                }, True)

        self.cmds = {
            '': self.showHelpSuggestion,
            'help': self.showHelp,
            'version': self.version,
            'config': self.configure,
            'start': self.start,
            'stop': self.killDaemon,
            'stats': self.showStats,
            'enable-plugin': self.enablePlugin,
            'enplugin': self.enablePlugin,
            'enableplugin': self.enablePlugin,
            'enmod': self.enablePlugin,
            'disable-plugin': self.disablePlugin,
            'displugin': self.disablePlugin,
            'disableplugin': self.disablePlugin,
            'dismod': self.disablePlugin,
            'reload-plugin': self.reloadPlugin,
            'reloadplugin': self.reloadPlugin,
            'reload-plugins': self.reloadPlugin,
            'reloadplugins': self.reloadPlugin,
            'listpeers': self.listPeers,
            'list-peers': self.listPeers,
            'addmsg': self.addMessage,
            'addmessage': self.addMessage,
            'add-msg': self.addMessage,
            'add-message': self.addMessage,
            'pm': self.sendEncrypt,
            'gui': self.openGUI,
            'addpeer': self.addPeer,
            'add-peer': self.addPeer,
            'add-address': self.addAddress,
            'addaddress': self.addAddress,
            'connect': self.addAddress
        }

        self.cmdhelp = {
            'help': 'Displays this Onionr help menu',
            'version': 'Displays the Onionr version',
            'config': 'Configures something and adds it to the file',
            'start': 'Starts the Onionr daemon',
            'stop': 'Stops the Onionr daemon',
            'stats': 'Displays node statistics',
            'enable-plugin': 'Enables and starts a plugin',
            'disable-plugin': 'Disables and stops a plugin',
            'reload-plugin': 'Reloads a plugin',
            'list-peers': 'Displays a list of peers',
            'add-peer': 'Adds a peer (?)',
            'add-msg': 'Broadcasts a message to the Onionr network',
            'pm': 'Adds a private message to block',
            'gui': 'Opens a graphical interface for Onionr'
        }

        command = ''
        try:
            command = sys.argv[1].lower()
        except IndexError:
            command = ''
        finally:
            self.execute(command)

        if not self._developmentMode:
            encryptionPassword = self.onionrUtils.getPassword(
                'Enter password to encrypt directory: ')
            self.onionrCore.dataDirEncrypt(encryptionPassword)
            shutil.rmtree('data/')

        return
Exemplo n.º 21
0
def setup_config(dataDir, o_inst=None):
    data_exists = os.path.exists(dataDir)
    if not data_exists:
        os.mkdir(dataDir)
    config.reload()

    if not os.path.exists(config._configfile):
        if os.path.exists('static-data/default_config.json'):
            # this is the default config, it will be overwritten if a config file already exists. Else, it saves it
            with open('static-data/default_config.json', 'r') as configReadIn:
                config.set_config(json.loads(configReadIn.read()))
        else:
            # the default config file doesn't exist, try hardcoded config
            logger.warn(
                'Default configuration file does not exist, switching to hardcoded fallback configuration!'
            )
            config.set_config({
                'dev_mode': True,
                'log': {
                    'file': {
                        'output': True,
                        'path': dataDir + 'output.log'
                    },
                    'console': {
                        'output': True,
                        'color': True
                    }
                }
            })

        config.save()

    settings = 0b000
    if config.get('log.console.color', True):
        settings = settings | logger.USE_ANSI
    if config.get('log.console.output', True):
        settings = settings | logger.OUTPUT_TO_CONSOLE
    if config.get('log.file.output', True):
        settings = settings | logger.OUTPUT_TO_FILE
    logger.set_settings(settings)

    if not o_inst is None:
        if str(config.get('general.dev_mode', True)).lower() == 'true':
            o_inst._developmentMode = True
            logger.set_level(logger.LEVEL_DEBUG)
        else:
            o_inst._developmentMode = False
            logger.set_level(logger.LEVEL_INFO)

    verbosity = str(config.get('log.verbosity', 'default')).lower().strip()
    if not verbosity in ['default', 'null', 'none', 'nil']:
        map = {
            str(logger.LEVEL_DEBUG): logger.LEVEL_DEBUG,
            'verbose': logger.LEVEL_DEBUG,
            'debug': logger.LEVEL_DEBUG,
            str(logger.LEVEL_INFO): logger.LEVEL_INFO,
            'info': logger.LEVEL_INFO,
            'information': logger.LEVEL_INFO,
            str(logger.LEVEL_WARN): logger.LEVEL_WARN,
            'warn': logger.LEVEL_WARN,
            'warning': logger.LEVEL_WARN,
            'warnings': logger.LEVEL_WARN,
            str(logger.LEVEL_ERROR): logger.LEVEL_ERROR,
            'err': logger.LEVEL_ERROR,
            'error': logger.LEVEL_ERROR,
            'errors': logger.LEVEL_ERROR,
            str(logger.LEVEL_FATAL): logger.LEVEL_FATAL,
            'fatal': logger.LEVEL_FATAL,
            str(logger.LEVEL_IMPORTANT): logger.LEVEL_IMPORTANT,
            'silent': logger.LEVEL_IMPORTANT,
            'quiet': logger.LEVEL_IMPORTANT,
            'important': logger.LEVEL_IMPORTANT
        }

        if verbosity in map:
            logger.set_level(map[verbosity])
        else:
            logger.warn(
                'Verbosity level %s is not valid, using default verbosity.' %
                verbosity)

    return data_exists
Exemplo n.º 22
0
    def __init__(self, debug):
        '''
            Initialize the api server, preping variables for later use

            This initilization defines all of the API entry points and handlers for the endpoints and errors
            This also saves the used host (random localhost IP address) to the data folder in host.txt
        '''

        config.reload()

        if config.get('devmode', True):
            self._developmentMode = True
            logger.set_level(logger.LEVEL_DEBUG)
        else:
            self._developmentMode = False
            logger.set_level(logger.LEVEL_INFO)

        self.debug = debug
        self._privateDelayTime = 3
        self._core = Core()
        self._crypto = onionrcrypto.OnionrCrypto(self._core)
        self._utils = onionrutils.OnionrUtils(self._core)
        app = flask.Flask(__name__)
        bindPort = int(config.get('client')['port'])
        self.bindPort = bindPort
        self.clientToken = config.get('client')['client_hmac']
        self.timeBypassToken = base64.b16encode(os.urandom(32)).decode()

        self.i2pEnabled = config.get('i2p')['host']

        self.mimeType = 'text/plain'

        with open('data/time-bypass.txt', 'w') as bypass:
            bypass.write(self.timeBypassToken)

        if not os.environ.get("WERKZEUG_RUN_MAIN") == "true":
            logger.debug('Your web password (KEEP SECRET): ' +
                         logger.colors.underline + self.clientToken)

        if not debug and not self._developmentMode:
            hostNums = [
                random.randint(1, 255),
                random.randint(1, 255),
                random.randint(1, 255)
            ]
            self.host = '127.' + str(hostNums[0]) + '.' + str(
                hostNums[1]) + '.' + str(hostNums[2])
        else:
            self.host = '127.0.0.1'
        hostFile = open('data/host.txt', 'w')
        hostFile.write(self.host)
        hostFile.close()

        @app.before_request
        def beforeReq():
            '''
                Simply define the request as not having yet failed, before every request.
            '''
            self.requestFailed = False

            return

        @app.after_request
        def afterReq(resp):
            if not self.requestFailed:
                resp.headers['Access-Control-Allow-Origin'] = '*'
            #else:
            #    resp.headers['server'] = 'Onionr'
            resp.headers['Content-Type'] = self.mimeType
            resp.headers[
                "Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'"
            resp.headers['X-Frame-Options'] = 'deny'
            resp.headers['X-Content-Type-Options'] = "nosniff"
            resp.headers['server'] = 'Onionr'

            # reset to text/plain to help prevent browser attacks
            if self.mimeType != 'text/plain':
                self.mimeType = 'text/plain'

            return resp

        @app.route('/client/')
        def private_handler():
            if request.args.get('timingToken') is None:
                timingToken = ''
            else:
                timingToken = request.args.get('timingToken')
            data = request.args.get('data')
            try:
                data = data
            except:
                data = ''
            startTime = math.floor(time.time())
            # we should keep a hash DB of requests (with hmac) to prevent replays
            action = request.args.get('action')
            #if not self.debug:
            token = request.args.get('token')

            if not self.validateToken(token):
                abort(403)
            self.validateHost('private')
            if action == 'hello':
                resp = Response('Hello, World! ' + request.host)
            elif action == 'shutdown':
                # request.environ.get('werkzeug.server.shutdown')()
                self.http_server.stop()
                resp = Response('Goodbye')
            elif action == 'ping':
                resp = Response('pong')
            elif action == 'stats':
                resp = Response('me_irl')
                raise Exception
            elif action == 'site':
                block = data
                siteData = self._core.getData(data)
                response = 'not found'
                if siteData != '' and siteData != False:
                    self.mimeType = 'text/html'
                    response = siteData.split(b'-', 2)[-1]
                resp = Response(response)
            else:
                resp = Response('(O_o) Dude what? (invalid command)')
            endTime = math.floor(time.time())
            elapsed = endTime - startTime

            # if bypass token not used, delay response to prevent timing attacks
            if not hmac.compare_digest(timingToken, self.timeBypassToken):
                if elapsed < self._privateDelayTime:
                    time.sleep(self._privateDelayTime - elapsed)

            return resp

        @app.route('/')
        def banner():
            self.mimeType = 'text/html'
            self.validateHost('public')
            try:
                with open('static-data/index.html', 'r') as html:
                    resp = Response(html.read())
            except FileNotFoundError:
                resp = Response("")
            return resp

        @app.route('/public/')
        def public_handler():
            # Public means it is publicly network accessible
            self.validateHost('public')
            action = request.args.get('action')
            requestingPeer = request.args.get('myID')
            data = request.args.get('data')
            try:
                data = data
            except:
                data = ''
            if action == 'firstConnect':
                pass
            elif action == 'ping':
                resp = Response("pong!")
            elif action == 'getHMAC':
                resp = Response(self._crypto.generateSymmetric())
            elif action == 'getSymmetric':
                resp = Response(self._crypto.generateSymmetric())
            elif action == 'getDBHash':
                resp = Response(self._utils.getBlockDBHash())
            elif action == 'getBlockHashes':
                resp = Response('\n'.join(self._core.getBlockList()))
            elif action == 'directMessage':
                resp = Response(self._core.handle_direct_connection(data))
            elif action == 'announce':
                if data != '':
                    # TODO: require POW for this
                    if self._core.addAddress(data):
                        resp = Response('Success')
                    else:
                        resp = Response('')
                else:
                    resp = Response('')
            # setData should be something the communicator initiates, not this api
            elif action == 'getData':
                if self._utils.validateHash(data):
                    if not os.path.exists('data/blocks/' + data + '.db'):
                        try:
                            resp = base64.b64encode(self._core.getData(data))
                        except TypeError:
                            resp = ""
                if resp == False:
                    abort(404)
                    resp = ""
                resp = Response(resp)
            elif action == 'pex':
                response = ','.join(self._core.listAdders())
                if len(response) == 0:
                    response = 'none'
                resp = Response(response)
            elif action == 'kex':
                peers = self._core.listPeers(getPow=True)
                response = ','.join(peers)
                resp = Response(response)
            else:
                resp = Response("")

            return resp

        @app.errorhandler(404)
        def notfound(err):
            self.requestFailed = True
            resp = Response("")

            return resp

        @app.errorhandler(403)
        def authFail(err):
            self.requestFailed = True
            resp = Response("403")

            return resp

        @app.errorhandler(401)
        def clientError(err):
            self.requestFailed = True
            resp = Response("Invalid request")

            return resp

        if not os.environ.get("WERKZEUG_RUN_MAIN") == "true":
            logger.info('Starting client on ' + self.host + ':' +
                        str(bindPort) + '...',
                        timestamp=True)

        try:
            self.http_server = WSGIServer((self.host, bindPort), app)
            self.http_server.serve_forever()
        except KeyboardInterrupt:
            pass
            #app.run(host=self.host, port=bindPort, debug=False, threaded=True)
        except Exception as e:
            logger.error(str(e))
            logger.fatal('Failed to start client on ' + self.host + ':' +
                         str(bindPort) + ', exiting...')
            exit(1)