def _generate_browser_caps(self, capabilities): caps = dict() chrome_options = webdriver.ChromeOptions() chrome_options.headless = os.environ.get( "RX_ENDPOINT_HEADLESS") == "True" if os.environ.get( "RX_DEBUG") == "True" chrome_options.add_argument("--auto-open-devtools-for-tabs") chrome_options.add_experimental_option( "prefs", {"devtools.preferences.currentDockState": "\"undocked\""}) caps["chrome"] = recursive_update(chrome_options.to_capabilities(), capabilities) caps["opera"] = recursive_update( {"operaOptions": { "binary": os.environ.get("RX_OPERA_PATH") }}, capabilities) caps["safari"] = capabilities caps["internet explorer"] = capabilities caps["MicrosoftEdge"] = capabilities firefox_options = webdriver.FirefoxOptions() firefox_options.headless = os.environ.get( "RX_ENDPOINT_HEADLESS") == "True" caps["firefox"] = recursive_update(firefox_options.to_capabilities(), capabilities) return caps
def test_all(self): self.assertEqual( recursive_update({'a': { 'b': 2 }}, { 'a': { 'b': 3, 'd': 4 }, 'e': 5 }), { 'a': { 'b': 3, 'd': 4 }, 'e': 5 }) self.assertEqual( recursive_update({'a': [1]}, { 'a': [2], 'c': { 'd': { 'c': 3 } } }), { 'a': [2], 'c': { 'd': { 'c': 3 } } }) self.assertEqual( recursive_update({ 'a': { 'c': 1, 'd': {} }, 'b': 4 }, {'b': 5}), { 'a': { 'c': 1, 'd': {} }, 'b': 5 }) self.assertEqual( recursive_update({ 'a': { 'c': 1, 'd': {} }, 'b': 4 }, {'a': 2}), { 'a': 2, 'b': 4 })
def main(): """ Demo how to run an agent """ dirname = os.path.dirname(__file__) if (len(sys.argv) > 1): env_configName = str(sys.argv[1]) agent_configName = str(sys.argv[2]) else: print("Default config ") env_configName = "./configs/factory_floor_experiment.yaml" env_filename = os.path.join(dirname, env_configName) agent_configName = "./configs/agent_combined_config.yaml" agent_filename = os.path.join(dirname, agent_configName) env_parameters = getParameters(env_filename) agent_parameters = getParameters(agent_filename) # whao, you need to know exact contents of all files here.. recursive_update( agent_parameters['subAgentList'][0]['parameters']['simulator'], env_parameters['environment']) print(env_parameters) print(agent_parameters) random.seed(env_parameters['seed']) maxSteps = env_parameters['max_steps'] baseEnv = FactoryFloor(env_parameters['environment']) packedActionSpace = PackedSpace(baseEnv.action_space, {"robots": ["robot1", "robot2", "robot3"]}) env = ModifiedGymEnv(baseEnv, packedActionSpace) logging.info("Starting example MCTS agent") logoutput = io.StringIO("episode output log") try: logoutputpickle = open('./' + os.environ["SLURM_JOB_ID"] + '.pickle', 'wb') except KeyError: print("No SLURM_JOB_ID found") logoutputpickle = io.BytesIO() obs = env.reset() complexAgent = createAgent(env.action_space, env.observation_space, agent_parameters) experiment = Experiment(complexAgent, env, maxSteps, render=True) experiment.addListener(JsonLogger(logoutput)) experiment.addListener(PickleLogger(logoutputpickle)) stats, confidence_ints = experiment.run() logoutputpickle.close() print("json output:", logoutput.getvalue()) print("\n\nREWARD STATS: " + str(stats) + " \nCONFIDENCE INTERVALS " + str(confidence_ints))
def __init__(self, config): """Create a driver with profile and repository configuration.""" config = recursive_update(self.default_config(), config) self.check_config(config) # The _config variables will never be changed self._config = config
def publish(*args, context=None, **kwargs): """ Notify jinja to publish the template to the output_file location with all of the context provided. :param args: the arguments that will be used to find the template in the template configuration :param context: :param kwargs: :return: """ # Build up the context argument, special kwarg context will be used to provide a starting dictionary if not context: context = {} recursive_update(context, kwargs) # Insert all of the site details into the context as well site_configuration = get_configuration().site.toDict() recursive_update(context.setdefault('site', {}), site_configuration) # Find the template definition object template_definition = _template_configuration.get('templates', {}) for template_path in args: try: template_definition = template_definition[template_path] except KeyError: print('Cannot find template definition: {} ' 'in template definitions: {}'.format( args, _template_configuration.get('templates', {}))) raise # Load the template and render to the destination file defined in the template_definition root_template = _environment.get_template( str(template_definition['template'])) output_file = template_definition['destination'].format(**context) output_content = root_template.render(context) output_file = _output_path / output_file # Verify that the path is possible and write out the file output_file.parent.mkdir(exist_ok=True) output_file.write_text(output_content) return output_file.relative_to(_output_path)
def __init__( self, name: str, period: str = None, start: dt.datetime = None, profile: Profile = None, targets: Dict[str, str] = None, description: str = "", delay: str = None, hooks: List[str] = None, queue: str = "default", configuration: Dict = None, ): # Check required arguments if period is None: raise ValueError("`period` is required.") if start is None: raise ValueError("`start` is required.") if profile is None: raise ValueError("`profile` is required.") if not targets: raise ValueError("`targets` is required.") # Set default values if delay is None: delay = period if hooks is None: hooks = [] if configuration is None: configuration = {} # Parse values self.delay = dtutil.parse_timedelta(delay) self.period = dtutil.parse_timedelta(period) self.name = name self.description = description self.start = dtutil.normalize_datetime(start) self.targets = targets self.queue = queue # Instantiate the driver driver_config = recursive_update(profile.configuration, configuration) for key, value in driver_config.items(): if isinstance(value, str): if value.startswith("env:$"): driver_config[key] = os.environ[value[5:]] self.driver = profile.driver(driver_config)
def get_config(): global CONFIG_DIR final_config = None if CONFIG_DIR is not None: env = os.environ[ENV_VARIABLE] default_config_path = os.path.join(CONFIG_DIR, 'default.json') specific_config_path = os.path.join(CONFIG_DIR, env + '.json') default_config, specific_config = None, None with open(default_config_path) as f: default_config = json.load(f) with open(specific_config_path) as f: specific_config = json.load(f) final_config = recursive_update(default_config, specific_config) return final_config
def __init__(self, agentId, actionspace:Dict, observationspace, parameters:dict): AtomicAgent.__init__(self, agentId, actionspace, observationspace, parameters) self._parameters = copy.deepcopy(self.DEFAULT_PARAMETERS) self._parameters = recursive_update(self._parameters, parameters) self._prev_state = None # TODO: change to self._step_output = dict({"obs": observation_space.sample(), "reward": None, "done": None, "prev_action": None}) self._step_output = None self._action = [-1] decoratedspace = DecoratedSpace.create(actionspace) self._num_actions = decoratedspace.n self._train_frequency = self._parameters['train_frequency'] self._save_frequency = self._parameters['save_frequency'] self._agentId = agentId self._PPO = PPO(self._parameters, self._num_actions) self._buffer = Buffer(self._parameters, self._num_actions) self._cumulative_rewards = 0 self._episode_step = 0 self._episodes = 1 self._t = 0 self._stats = {"cumulative_rewards": [], "episode_length": [], "value": [], "learning_rate": [], "entropy": [], "policy_loss": [], "value_loss": []} tf.reset_default_graph() self._step = 0 summary_path = 'summaries/' + self._parameters['name'] + '_' + \ self._parameters['algorithm'] if not os.path.exists(summary_path): os.makedirs(summary_path) self._summary_writer = tf.summary.FileWriter(summary_path) if self._parameters['influence']: self._seq_len = self._parameters['inf_seq_len'] elif self._parameters['recurrent']: self._seq_len = self._parameters['seq_len'] else: self._seq_len = 1
def _collect(self): merged = {} for m in self._maps.values(): merged = recursive_update(merged, m) return frozendict(merged)
def __init__(self, agentId, actionspace: Dict, observationspace, parameters: dict): """ @param parameters dict that must contain keys 'otherAgents', 'treeAgent' and 'rolloutAgent' 'otherAgents' must map to a (possibly empty) list of dict objects for a call to createAgents 'treeAgent' and 'rolloutAgent' must map to a dict object for a call to createAgent. The dict must also contain a 'simulator' key containing a copy of the env parameters, so that the agent can create a duplicate environment. The simulator dict must contain a key 'fullname' containing the full name of the environment for the class loader (se EnvironmentsFactory). """ super().__init__(agentId, actionspace, observationspace, parameters) if not ('treeAgent' in parameters and 'rolloutAgent' in parameters): raise "parameters does not contain 'treeAgent', 'rolloutAgent':" + str( parameters) self._parameters = copy.deepcopy(self.DEFAULT_PARAMETERS) self._parameters = recursive_update(self._parameters, parameters) if 'timeLimit' in self._parameters: if 'iterationLimit' in self._parameters: raise ValueError( "Cannot have both a time limit and an iteration limit") self._limitType = 'time' else: if 'iterationLimit' not in self._parameters: DEFAULT_LIMIT = 1000 logging.error( "Must have either a time limit or an iteration limit. Using default iteration limit: " + str(DEFAULT_LIMIT)) self._parameters['iterationLimit'] = DEFAULT_LIMIT # number of iterations of the search if self._parameters['iterationLimit'] < 1: raise ValueError("Iteration limit must be greater than one") self._limitType = 'iterations' # start the simulator environment envparams = self._parameters['simulator'] e = EnvironmentFactory.createEnvironment(envparams['fullname'], envparams) self._simulator = ModifiedGymEnv( e, DecoratedSpace.create(copy.deepcopy(e.action_space))) # diyBonus logic: to refactor -- include in a simulator factory / only for FactoryFloor env diyBonus = self._parameters.get("diyBonus") if diyBonus is not None: self._simulator = DiyFactoryFloorAdapter(self._simulator, diyBonus, self.agentId) self._treeAgent = createAgent(self._simulator.action_space, self._simulator.observation_space, parameters['treeAgent']) if 'otherAgents' in parameters: rolloutAgentDict = copy.deepcopy(parameters['otherAgents']) rolloutAgentList = rolloutAgentDict['subAgentList'] rolloutAgentList.append(parameters['rolloutAgent']) rolloutAgentDict['subAgentList'] = rolloutAgentList self._rolloutAgent = createAgent(self._simulator.action_space, self._simulator.observation_space, rolloutAgentDict) self._otherAgents = createAgent(self._simulator.action_space, self._simulator.observation_space, parameters['otherAgents']) else: self._otherAgents = None self._rolloutAgent = createAgent(self._simulator.action_space, self._simulator.observation_space, parameters['rolloutAgent'])
def setUpClass(cls): logger.log( 60, '#################### Testplan %s - START ####################' % cls.__name__) logging.getLogger('beehive.test.run')\ .log(60, '#################### Testplan %s - START ####################' % cls.__name__) self = cls # ssl path = os.path.dirname(__file__).replace('beehive/common', 'beehive/tests') pos = path.find('tests') path = path[:pos + 6] keyfile = None certfile = None # load configs try: home = os.path.expanduser('~') if self.main_config_file is None: config_file = '%s/beehive.yml' % home self.main_config_file = config_file else: config_file = self.main_config_file config = self.load_file(config_file) logger.info('Get beehive test configuration: %s' % config_file) except Exception as ex: raise Exception( 'Error loading config file. Search in user home. %s' % ex) # load configs fernet key try: home = os.path.expanduser('~') if self.main_config_file is None: config_file = '%s/beehive.fernet' % home self.main_fernet_file = config_file else: config_file = self.main_config_file.replace('yml', 'fernet') fernet = self.load_file(config_file) logger.info('Get beehive test fernet key: %s' % config_file) except Exception as ex: raise Exception( 'Error loading fernet key file. Search in user home. %s' % ex) # load specific configs for a set of test try: if self.spec_config_file is not None: config2 = self.load_file(self.spec_config_file) recursive_update(config, config2) logger.info('Get beehive test specific configuration: %s' % self.spec_config_file) except Exception as ex: raise Exception( 'Error loading config file. Search in user home. %s' % ex) logger.info('Validation active: %s' % cls.validation_active) cfg = config self.test_config = config.get('configs', {}) if self.test_config.get('resource', None) is not None: for key in self.test_config.get('resource').keys(): if 'configs' in cfg.keys() and 'resource' in cfg.get( 'configs').keys(): self.test_config.get('resource').get(key).update( cfg.get('configs').get('resource').get(key, {})) if 'configs' in cfg.keys() and 'container' in cfg.get( 'configs').keys(): self.test_config.get('container').update( cfg.get('configs').get('container')) self.fernet = fernet # endpoints self.endpoints = cfg.get('endpoints') self.swagger_endpoints = cfg.get('swagger') logger.info('Endpoints: %s' % self.endpoints) # redis connection if cfg.get('redis') is not None: self.redis_uri = cfg.get('redis').get('uri') if self.redis_uri is not None and self.redis_uri != '': rhost, rport, db = self.redis_uri.split(';') self.redis = redis.StrictRedis(host=rhost, port=int(rport), db=int(db)) # celery broker self.worker = cfg.get('worker') # mysql connection self.db_uris = cfg.get('db-uris') # get users self.users = cfg.get('users') # create auth client self.auth_client = BeehiveApiClient([], 'keyauth', None, '', None) # create api endpoint self.api = {} self.schema = {} for subsystem, endpoint in self.endpoints.items(): self.api[subsystem] = RemoteClient(endpoint, keyfile=keyfile, certfile=certfile) self.load_result() self.custom_headers = {} self.endpoit_service = 'auth'
def make_message(self, d): self.payload = {} [ recursive_update(self.payload, self.dot_expand(key, d[key])) for key in d ]
def run_experiment(run_name, params=None): exp = NIC_Experiment('stroke_mri_segmentation', run_name, device_name='cuda') os.environ['CUDA_VISIBLE_DEVICES'] = '0' print("Using GPU {}".format(os.environ['CUDA_VISIBLE_DEVICES'])) # -------------------------------------------------------------- # CONFIGURATION # -------------------------------------------------------------- cfg = { # Dataset 'dataset': None, 'architecture': None, # Patches 'npatches': 10000, 'patch_shape': (24, 24, 16), 'ratio_lesion': 0.5, # Training 'do_train': True, 'batch_size': 16, 'train_loss': NIC_binary_focal_loss(gamma=2.0, alpha=0.25), 'early_stop': { 'patience': 8, 'metric': 'l1_er' }, # Evaluation 'num_folds': 4, 'crossval_folds': None, # Prediction 'patch_shape_pred': (24, 24, 16), 'pred_extraction_step': (4, 4, 1), 'uncertainty': { 'runs': 0, 'rate': 0.1, 'type': 'alpha' } } if params is not None: print("Setting parameters in configuration queue") recursive_update( cfg, params ) # Overwrite cfg with keys present in paramas (others left untouched) # Other parameters thresholds = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] lesion_sizes = [1, 10, 50, 100, 200, 300, 500, 1000] # -------------------------------------------------------------- # Experiment # -------------------------------------------------------------- # 1. Dataset assert isinstance(cfg['dataset'], NIC_Dataset) dataset = copy.copy(cfg['dataset']) dataset.load() print("Loaded {} dataset with {} training and {} testing".format( dataset.__class__.__name__, len(dataset.train), len(dataset.test))) # 2. Model model_def = copy.deepcopy(cfg['architecture']) model_parameters = filter(lambda p: p.requires_grad, model_def.parameters()) nparams = sum([np.prod(p.size()) for p in model_parameters]) print("Setting model instance from {} architecture with {} parameters". format(model_def.__class__.__name__, nparams)) # 3. Training and validation sets train_gen = PatchGeneratorBuilder( batch_size=cfg['batch_size'], zeropad_shape=cfg['patch_shape'], instruction_generator=PatchInstructionGenerator( in_shape=cfg['patch_shape'], out_shape=cfg['patch_shape'], sampler=StrokeLesionSampling(in_shape=cfg['patch_shape'], num_patches=cfg['npatches'], ratio_lesion=cfg['ratio_lesion'], augment_factor=2), augment_to=cfg['npatches']), num_workers=4, shuffle=True) val_gen = PatchGeneratorBuilder( batch_size=cfg['batch_size'], zeropad_shape=cfg['patch_shape'], instruction_generator=PatchInstructionGenerator( in_shape=cfg['patch_shape'], out_shape=cfg['patch_shape'], sampler=StrokeLesionSampling(in_shape=cfg['patch_shape'], num_patches=cfg['npatches'], ratio_lesion=cfg['ratio_lesion'], augment_factor=2), augment_to=cfg['npatches']), num_workers=4, shuffle=True) trainer = EarlyStoppingTrain( do_train=cfg['do_train'], device=exp.get_device(), max_epochs=200, batch_size=cfg['batch_size'], loss_func=cfg['train_loss'], optimizer=TorchOptimizer(torch.optim.Adadelta, opts={'rho': 0.95}), train_metrics={'acc': nic_binary_accuracy}, eval_metrics={ 'acc': nic_binary_accuracy, 'l1_er': nic_binary_l1_er }, early_stopping_metric=cfg['early_stop']['metric'], early_stopping_patience=cfg['early_stop']['patience']) predictor = PatchPredictor( device=exp.get_device(), num_classes=2, lesion_class=1, uncertainty_passes=cfg['uncertainty']['runs'], uncertainty_dropout=cfg['uncertainty']['rate'], uncertainty_dotype=cfg['uncertainty']['type'], zeropad_shape=cfg['patch_shape_pred'], instruction_generator=PatchGeneratorBuilder( batch_size=64, shuffle=False, zeropad_shape=None, instruction_generator=PatchInstructionGenerator( in_shape=cfg['patch_shape_pred'], out_shape=cfg['patch_shape_pred'], sampler=UniformSampling( in_shape=cfg['patch_shape_pred'], extraction_step=cfg['pred_extraction_step']), augment_to=None))) crossval = SimpleCrossvalidation( model_definition=model_def, images=dataset.train, num_folds=cfg['num_folds'], model_trainer=trainer, train_instr_gen=train_gen, val_instr_gen=val_gen, checkpoint_pathfile=os.path.join( exp.checkpoints_dir, '{}.pt'.format(run_name)), # '{}.pt'.format(run_name) log_pathfile=exp.get_log_filename(), test_predictor=predictor, results_path=os.path.join(exp.results_dir, '{}/'.format(run_name)), ) # EXECUTION # Run validation crossval.run_crossval() # Load probability maps results = load_result_set(os.path.join(exp.results_dir, '{}/'.format(run_name)), dataset.train, result_type='pred', file_format='nii.gz') # Grid search of post-processing options metrics_list, metrics_names = thresh_size_search(results, dataset.train, thresholds, lesion_sizes) print_metrics_list(metrics_list, metrics_names) save_metrics_list( os.path.join(exp.metrics_dir, '{}_{}.csv'.format(get_formatted_timedate(), run_name)), metrics_list, metrics_names)