Ejemplo n.º 1
0
def do_nilsson(n_max, omega, parity, user_pars):
    """
  Find the energies and eigenstates for the Nilsson model, with the given Omega(=Jz) and parity (0=even, 1=odd).
  Energies are reported in units of omega00=(42 MeV)/A^(1/3).
  User_pars is a hash such as {'kappa':0.06,'mu':0.5,'delta':0.2}. Sane defaults are provided for all parameters for
  testing purposes, and these parameters are also given back in the returned hash.
  Returns a hash with keys n_states,states,index,evals,evecs,ham.
  """
    if omega % 2 == 0:
        raise Excaption("even value of Omega in do_nilsson")
    space = (n_max, omega, parity)
    index = hamiltonian.enumerate_states(
        space
    )  # dictionary mapping from Nilsson quantum numbers to array indices
    states = util.dict_to_tuple(util.invert_dict(
        index))  # inverse of index, maps from index to quantum numbers
    n_states = len(states)
    default_pars = {'kappa': 0.06, 'mu': 0.5, 'delta': 0.0}
    pars = util.merge_dicts(default_pars, user_pars)
    ham = hamiltonian.hamiltonian(space, pars, index, states)
    evals, evecs = hamiltonian.eigen(ham)
    return util.merge_dicts(
        pars, {
            'n_states': n_states,
            'states': states,
            'index': index,
            'evals': evals,
            'evecs': evecs,
            'ham': ham
        })
Ejemplo n.º 2
0
def load_config():
    home_config = load_config_from_user_home()
    env_config = load_config_from_env()
    merged = util.merge_dicts(home_config, env_config)
    cli_cfg = load_config_from_args()
    merged = util.merge_dicts(merged, cli_cfg)
    add_config_source(merged)
Ejemplo n.º 3
0
def export_config_for_file(filename):
    dir_config = export_config_for_directory(os.path.dirname(filename), "osx")
    bn = os.path.basename(filename)
    for k in dir_config.keys():
        if fnmatch.fnmatch(k, bn):
            file_dict = dir_config[k]
            util.merge_dicts(dir_config, file_dict)
    return dir_config
Ejemplo n.º 4
0
    def _calculate_baseline_oracle(self, label: str) -> List[Stats]:
        p = len(self.splits[label]['y_train'][
            self.splits[label]['y_train'] == True]) / len(
                self.splits[label]['y_train'])
        out = {'label': label, 'prevalence': p}

        # oracle
        clf = RandomForestClassifier(n_estimators=self.clf_n_estimator,
                                     n_jobs=-1,
                                     random_state=self.random_seed)
        start = time()
        clf.fit(self.splits[label]['x_train'], self.splits[label]['y_train'])
        elapsed = time() - start
        predictions = clf.predict(self.splits[label]['x_dev'])
        scores = clf.predict_proba(self.splits[label]['x_dev'])[:, 1]
        oracle = self._get_metrics(self.splits[label]['y_dev'], predictions,
                                   scores)
        oracle = util.merge_dicts(oracle, {'train time (s)': elapsed})
        oracle = util.add_prefix_to_dict_keys(oracle, 'oracle_')
        out = util.merge_dicts(out, oracle)

        # baselines
        # random with same prevalence
        prevalence = self.splits[label]['y_train'].value_counts(normalize=True)
        categories = np.array(prevalence.index).astype(bool)
        n = len(self.splits[label]['x_dev'])
        baseline_random = pd.DataFrame([
            self._get_metrics(actual=self.splits[label]['y_dev'],
                              predicted=np.random.choice(categories,
                                                         p=prevalence,
                                                         size=n))
            for _ in range(self.baseline_random_n)
        ]).median().to_dict()
        baseline_random = util.add_prefix_to_dict_keys(baseline_random,
                                                       'baseline_random_')
        out = util.merge_dicts(out, baseline_random)

        # majority
        # majority_class = prevalence.index[0]
        # baseline_majority = self._get_metrics(actual=self.splits[label]['y_dev'],
        #                                       predicted=[majority_class] * n)
        # baseline_majority = util.add_prefix_to_dict_keys(baseline_majority, 'baseline_majority_')
        # out = util.merge_dicts(out, baseline_majority)

        # unsupervised
        iforest = IsolationForest(contamination=prevalence[True],
                                  behaviour='new',
                                  n_estimators=self.clf_n_estimator,
                                  random_state=self.random_seed)
        iforest.fit(self.splits[label]['x_train'])
        baseline_unsupervised = self._get_metrics(
            actual=self.splits[label]['y_dev'],
            predicted=iforest.predict(self.splits[label]['x_dev']) == -1)
        baseline_unsupervised = util.add_prefix_to_dict_keys(
            baseline_unsupervised, 'baseline_unsupervised_')
        out = util.merge_dicts(out, baseline_unsupervised)

        return [out]
Ejemplo n.º 5
0
def configure_user(config, args):
    config_user = dict()
    if os.path.exists("config.user.jsn"):
        config_user = jsn.loads(open("config.user.jsn", "r").read())
    if util.get_platform_name() == "win32":
        if "-msbuild" not in sys.argv:
            configure_vc_vars_all(config_user)
            configure_windows_sdk(config_user)
    if os.path.exists("config.user.jsn"):
        config_user = jsn.loads(open("config.user.jsn", "r").read())
        util.merge_dicts(config, config_user)
Ejemplo n.º 6
0
    def update_configs(new):
        """
        Add new configurations, but keep users' changes intact. This will
        have the wrong version number, which will have to be overwritten
        later
        """
        sc = SC()
        new_dict = json.loads(new)

        # map old configs on top of new to retain user settings
        merge_dicts(new_dict, sc.dict)
        sc.dict = new_dict
        sc.save_dict()
Ejemplo n.º 7
0
    def update_configs(new):
        """
        Add new configurations, but keep users' changes intact. This will
        have the wrong version number, which will have to be overwritten
        later
        """
        sc = SC()
        new_dict = json.loads(new)

        # map old configs on top of new to retain user settings
        merge_dicts(new_dict, sc.dict)
        sc.dict = new_dict
        sc.save_dict()
Ejemplo n.º 8
0
def export_config_for_directory(filedir, platform):
    filepath = util.sanitize_file_path(filedir)
    dirtree = filepath.split(os.sep)
    export_dict = dict()
    subdir = ""
    for i in range(0, len(dirtree)):
        subdir = os.path.join(subdir, dirtree[i])
        export = os.path.join(subdir, "export.jsn")
        if os.path.exists(export):
            dir_dict = jsn.loads(open(export, "r").read())
            util.merge_dicts(export_dict, dir_dict)
    if platform in export_dict.keys():
        util.merge_dicts(export_dict, export_dict[platform])
    return export_dict
Ejemplo n.º 9
0
    def cfg_entry_for_branch(self, branch):
        matching_entries = [
            entry for entry in self.cfg_entries()
            if entry.branch_matches(branch)
        ]
        if not matching_entries:
            return None

        # merge entries
        effective_branch_cfg = {}

        # order by optional attribute 'index' (random order if omitted)
        def entry_with_idx(idx, entry):
            if entry.index():
                idx = int(entry.index())
            return (idx, entry)

        indexed_entries = [
            entry_with_idx(idx, entry)
            for idx, entry in enumerate(matching_entries)
        ]

        for _, entry in sorted(indexed_entries, key=lambda i: i[0]):
            effective_branch_cfg = merge_dicts(effective_branch_cfg, entry.raw)

        return BranchCfgEntry(name='merged', raw_dict=effective_branch_cfg)
Ejemplo n.º 10
0
    def _render(self, definition_descriptor):
        effective_definition = definition_descriptor.pipeline_definition

        # handle inheritance
        for override in definition_descriptor.override_definitions:
            effective_definition = merge_dicts(effective_definition, override)

        template_name = definition_descriptor.template_name()
        template_contents = self.template_retriever.template_contents(template_name)

        pipeline_name = definition_descriptor.pipeline_name

        # support declaring jobs by either 'jobs' or 'variants'
        # TODO: Add deprecation message for old 'variants' syntax.
        jobs = effective_definition.get('jobs', {})
        variants = effective_definition.get('variants', {})
        if jobs and variants:
            raise RuntimeError(
                f"Both 'jobs' and 'variants' are defined in pipeline '{pipeline_name}'"
            )

        pipeline_definition = RawPipelineDefinitionDescriptor(
            name=pipeline_name,
            base_definition=effective_definition.get('base_definition', {}),
            variants=jobs or variants,
            template=template_name,
        )

        factory = DefinitionFactory(raw_definition_descriptor=pipeline_definition)
        pipeline_metadata = dict()
        pipeline_metadata['definition'] = factory.create_pipeline_definition()
        pipeline_metadata['name'] = pipeline_definition.name
        pipeline_metadata['target_team'] = definition_descriptor.concourse_target_team
        generated_model = pipeline_metadata.get('definition')

        # determine pipeline name (if there is main-repo, append the configured branch name)
        for variant in pipeline_metadata.get('definition').variants():
            # hack: take the first "main_repository" we find
            if not variant.has_main_repository():
                continue
            main_repo = variant.main_repository()
            pipeline_metadata['pipeline_name'] = '-'.join(
                [pipeline_definition.name, main_repo.branch()]
            )
            break
        else:
            # fallback in case no main_repository was found
            pipeline_metadata['pipeline_name'] = pipeline_definition.name
            main_repo = None

        t = mako.template.Template(template_contents, lookup=self.lookup)

        definition_descriptor.pipeline = t.render(
                instance_args=generated_model,
                config_set=self.cfg_set,
                pipeline=pipeline_metadata,
        )

        return definition_descriptor
Ejemplo n.º 11
0
def test_merge_dicts(input_pt: dict):
    """Testa a mesclagem de dicionários.
    """
    input1 = input_pt.copy()
    input2 = input_pt.copy()

    del input1["atividades"][0]["qtde_entregas"]
    del input2["atividades"][0]["avaliacao"]

    assert util.merge_dicts(input1, input2) == input_pt
Ejemplo n.º 12
0
    def test_merge_dicts_with_merge_retains_order(self):
        left = {1: [3, 1, 0]}
        right = {1: [1, 2, 4]}

        merged = examinee.merge_dicts(left, right, list_semantics='merge')

        self.assertEqual(
            merged,
            {1: [3, 1, 0, 2, 4]},
        )
Ejemplo n.º 13
0
    def __init__(self,
                 learner,
                 learner_kwargs,
                 agent_kwargs,
                 folds,
                 data_info,
                 agent=WarmLearnerAgent,
                 subset_prop=1,
                 feature_subset=None,
                 **kwargs):
        super(CrossValidationAgent, self).__init__(**kwargs)

        self.data_info = data_info

        self.child_info = []
        for train, test in folds:
            if subset_prop < 1:
                # noinspection PyUnresolvedReferences
                train = train[:int(np.floor(subset_prop * train.size))]
            self.child_info.append((agent,
                                    util.merge_dicts(
                                        agent_kwargs,
                                        dict(learner=learner,
                                             learner_kwargs=learner_kwargs,
                                             train_idx=train,
                                             test_idx=test,
                                             data_info=data_info,
                                             feature_subset=feature_subset))))

        self.score_times = []
        self.score_values = []
        self.held_out_prediction_times = []
        self.held_out_prediction_files = []
        self.valid_prediction_times = []
        self.valid_prediction_files = []
        self.test_prediction_times = []
        self.test_prediction_files = []

        self.child_score_times = dict()
        self.child_score_values = dict()
        self.child_held_out_prediction_times = dict()
        self.child_held_out_prediction_files = dict()
        self.child_held_out_idx = dict()
        self.child_valid_prediction_times = dict()
        self.child_valid_prediction_files = dict()
        self.child_test_prediction_times = dict()
        self.child_test_prediction_files = dict()

        self.communication_sleep = 0.1

        # TODO: Improve this hack!
        if agent == WarmLearnerAgent:
            self.immortal_offspring = True
        else:
            self.immortal_offspring = False
Ejemplo n.º 14
0
    def _inject_main_repo(self, base_definition, repo_path, branch_name):
        main_repo_raw = {'path': repo_path, 'branch': branch_name}

        if base_definition.get('repo'):
            merged_main_repo = merge_dicts(base_definition['repo'],
                                           main_repo_raw)
            base_definition['repo'] = merged_main_repo
        else:
            base_definition['repo'] = main_repo_raw

        return base_definition
Ejemplo n.º 15
0
    def test_merge_dicts_simple(self):
        left = {1: {2: 3}}
        right = {1: {4: 5}, 6: 7}

        merged = examinee.merge_dicts(left, right)

        self.assertEqual(
            merged,
            {
                1: {2: 3, 4: 5},
                6: 7,
            }
        )
Ejemplo n.º 16
0
def main(model, source_split, mask_split, mode, index_range,
         extra_sbatch_params, extra_env_vars):
    assert model in SBATCH_PARAMS_EXTRA
    assert model in MODEL_NAME_RUN_PATH_MAP
    assert model in MODEL_NAME_IMAGE_SIZE_DIVISOR_MAP

    dataset_name = f'{source_split}_{mask_split}'

    env = {
        'RUN_PATH':
        MODEL_NAME_RUN_PATH_MAP[model],
        'IMAGE_SIZE_DIVISOR':
        MODEL_NAME_IMAGE_SIZE_DIVISOR_MAP[model],
        'FRAMES_DATASET_PATH':
        os.path.join('datasets', 'devil', f'{source_split}.tar'),
        'MASKS_DATASET_PATH':
        os.path.join('datasets', 'devil', f'{mask_split}.tar'),
        'INPAINTING_RESULTS_ROOT':
        os.path.join('inpainting-results', 'devil', dataset_name, model),
        'TEMP_ROOT':
        '/tmp',
        'INDEX_RANGE':
        '' if index_range is None else f'{index_range[0]}-{index_range[1]}'
    }
    for env_var_name, env_var_value in zip(extra_env_vars[::2],
                                           extra_env_vars[1::2]):
        env[env_var_name] = env_var_value

    if mode == 'local':
        check_call(
            ['bash', os.path.join(SCRIPT_DIR, 'inpaint-videos.sh')], env=env)
    else:
        job_name = f'inpaint-videos-{model}-{dataset_name}'
        if index_range is not None:
            job_name += f'_{index_range[0]}-{index_range[1]}'
        basic_sbatch_params = {
            'job-name': job_name,
            'nodes': 1,
            'output': f'logs/%j-%N-{job_name}.out',
            'error': f'logs/%j-%N-{job_name}.out',
            'mail-user': f'{os.environ["USER"]}@umich.edu',
            'mail-type': 'ALL',
        }
        sbatch_params = merge_dicts(basic_sbatch_params,
                                    SBATCH_PARAMS_EXTRA[model])

        for param_name, param_value in zip(extra_sbatch_params[::2],
                                           extra_sbatch_params[1::2]):
            sbatch_params[param_name] = param_value

        sbatch('inpaint-videos.sh', sbatch_params, SCRIPT_DIR, env)
Ejemplo n.º 17
0
 def _active_learning_initial_training(
     self, semi_sup: bool, stats: Stats, data_for_plotting: List[Stats],
     learner: Optional[BaseEstimator], sampling_strategy: Callable,
     active_learning_data: ActiveLearningData, labeled_indices: List[int]
 ) -> Tuple[ActiveSemiSup, Stats, List[Stats]]:
     if semi_sup:
         clf = self.semi_supervised_class(**self.semi_supervised_class_args)
         x, y = self._construct_semi_supervised_data(
             active_learning_data.x_train_start,
             active_learning_data.y_train_start,
             active_learning_data.x_train_pool,
             active_learning_data.y_train_pool, labeled_indices)
         clf, elapsed_train = util.timer(clf.fit, **{'X': x, 'y': y})
     else:
         clf, elapsed_train = util.timer(
             ActiveLearner,
             **dict(estimator=learner,
                    query_strategy=sampling_strategy,
                    X_training=active_learning_data.x_train_start.values,
                    y_training=active_learning_data.y_train_start.values))
     predicted, elapsed_query = util.timer(
         clf.predict, **{'X': active_learning_data.x_dev})
     predicted = clf.predict(active_learning_data.x_dev)
     # [:, 1] to get positive class probabilities, semi-sup probabilities can be NaN so skip
     scores = None if semi_sup else clf.predict_proba(
         active_learning_data.x_dev)[:, 1]
     metrics = self._get_metrics(actual=active_learning_data.y_dev,
                                 predicted=predicted,
                                 scores=scores)
     data_for_plotting.append(
         self._get_plotting_row(-1, metrics, elapsed_train, elapsed_query))
     metrics = util.add_prefix_to_dict_keys(metrics, 'initial_')
     stats = util.merge_dicts(stats, {
         'train time (s)': elapsed_train,
         'query time (s)': elapsed_query
     })
     stats = util.merge_dicts(stats, metrics)
     return clf, stats, data_for_plotting
Ejemplo n.º 18
0
    def _scan_repository_for_definitions(
        self,
        repository,
        github_cfg,
        org_name,
    ) -> RawPipelineDefinitionDescriptor:
        for branch_name, cfg_entry in self._determine_repository_branches(
                repository=repository):
            try:
                definitions = repository.file_contents(
                    path='.ci/pipeline_definitions', ref=branch_name)
            except NotFoundError:
                continue  # no pipeline definition for this branch

            repo_hostname = urlparse(github_cfg.http_url()).hostname
            override_definitions = cfg_entry.override_definitions(
            ) if cfg_entry else {}

            verbose('from repo: ' + repository.name + ':' + branch_name)
            try:
                definitions = yaml.load(definitions.decoded.decode('utf-8'),
                                        Loader=yaml.SafeLoader)
            except BaseException as e:
                repo_path = f'{org_name}/{repository.name}'
                yield DefinitionDescriptor(
                    pipeline_name='<invalid YAML>',
                    pipeline_definition={},
                    main_repo={
                        'path': repo_path,
                        'branch': branch_name,
                        'hostname': repo_hostname
                    },
                    concourse_target_cfg=self.cfg_set.concourse(),
                    concourse_target_team=self.job_mapping.team_name(),
                    override_definitions=(),
                    exception=e,
                )
                return  # nothing else to yield in case parsing failed

            # handle inheritance
            definitions = merge_dicts(definitions, override_definitions)

            yield from self._wrap_into_descriptors(
                repo_path='/'.join([org_name, repository.name]),
                repo_hostname=repo_hostname,
                branch=branch_name,
                raw_definitions=definitions,
                override_definitions=override_definitions,
            )
Ejemplo n.º 19
0
 def update_stats(stats_, data_for_plotting_, prefix, x_, y_, i_):
     learner.fit(x_, y_)
     predicted = learner.predict(x_dev)
     scores = learner.predict_proba(x_dev)[:, 1]
     metrics = self._get_metrics(actual=active_learning_data.y_dev,
                                 predicted=predicted,
                                 scores=scores)
     data_for_plotting_.append(
         self._get_plotting_row(i_,
                                metrics,
                                elapsed_train=0,
                                elapsed_query=0))
     metrics = util.add_prefix_to_dict_keys(metrics, prefix)
     stats_ = util.merge_dicts(stats_, metrics)
     return stats_, data_for_plotting_
Ejemplo n.º 20
0
 def print_papers_to_all_binded_devices(self, user_id: str, papers: List[Paper]):
     """
     向给定咕咕号下所有绑定的设备打印纸条
     :type papers: 要打印的纸条
     :param user_id: 咕咕号
     :return: 打印的纸条序列号和对应的纸条以及打印状态
     """
     result_dic: Dict[int, Paper] = dict()
     if not self.users.__contains__(user_id):
         raise Exception("查询不到给定咕咕号")
     devices: Dict[str, int] = self.users[user_id]
     for each_device in devices:
         each_result = self.print_papers(papers=papers, user_id=user_id, device_id=each_device)
         result_dic = merge_dicts(result_dic, each_result)
     return result_dic
Ejemplo n.º 21
0
    def _create_variants_dict(self, raw_definition_descriptor):
        variants_dict = normalise_to_dict(
            deepcopy(raw_definition_descriptor.variants))

        base_dict = deepcopy(raw_definition_descriptor.base_definition)

        merged_variants = {}
        for variant_name, variant_args in variants_dict.items():
            # optimisation: if there are no variant-specific arguments, we do not need to merge
            if variant_args:
                merged_variants[variant_name] = merge_dicts(
                    base_dict, variant_args)
            else:
                merged_variants[variant_name] = deepcopy(base_dict)

        return merged_variants
    def get_concept_terms(self, concept_type):
        #### INPUT = 'concept type'
        #### OUTPUT = object_tree containing all the snomed node objects
        # NOTE - is used to simplify the call to get_related_cuis, which has more flexibility, but is more complex
        if concept_type.lower() == 'negation':
            object_tree = self.get_related_cuis(destination_id='272519000', target_relation='116680003',
                                                inverse=1)
            ### Manually add in 'Normal', which isn't in negation finding hierarchy
            normal_node = node()
            normal_node.term='Normal (qualifier value)'
            normal_node.id='17621005'
            object_tree['17621005'] = normal_node

        elif concept_type.lower() == 'qualifier':
            object_tree = self.get_related_cuis(destination_id='362981000', target_relation='116680003',
                                                inverse=1)
        elif concept_type.lower() == 'qualifier_minimal':
            ## SNOMED counts a LOT (e.g. languages) as qualifiers, so this returns a smaller, more common usage list of
            ## the sub-qualifiers that are most applicable to common language
            ### Look into making a customized qualifer list (try mix of Finding values + Finding status values +

            # General information qualifier, + Result Comments + Ranked categories + Descriptor +
            # Modifier mainly for procedure + special information qualifier)
            gen_info_qual_tree = self.get_related_cuis(destination_id='106227002',
                                                       target_relation='116680003', inverse=1)
            result_comments_tree = self.get_related_cuis(target_relation='116680003',
                                                         destination_id='281296001', inverse=1)
            ranked_categories_tree = self.get_related_cuis(target_relation='116680003',
                                                           destination_id='272396007', inverse=1)
            descriptor_tree = self.get_related_cuis(target_relation='116680003', destination_id='272099008',
                                                    inverse=1)
            modifier_tree = self.get_related_cuis(target_relation='116680003', destination_id='106239005',
                                                  inverse=1)
            special_info_qual_tree = self.get_related_cuis(target_relation='116680003',
                                                           destination_id='106239005', inverse=1)
            from util import merge_dicts
            object_tree = merge_dicts(gen_info_qual_tree, result_comments_tree, ranked_categories_tree,
                                      descriptor_tree, modifier_tree, special_info_qual_tree)
            # object_tree.update(gen_info_qual_tree).update(result)



        else:
            return NameError
        for this_node in object_tree:
            object_tree[this_node].concept_type=concept_type
        self.concept_network_dictionary=object_tree
Ejemplo n.º 23
0
 def _get_metrics(
         actual: Iterable,
         predicted: Iterable,
         scores: Optional[Iterable[float]] = None) -> Dict[str, float]:
     tn, fp, fn, tp = confusion_matrix(actual, predicted).ravel()
     metrics = {
         'precision': precision_score(actual, predicted),
         'recall': recall_score(actual, predicted),
         'f1': f1_score(actual, predicted),
         'FP': fp,
         'FN': fn
     }
     metrics_threshold = {} if scores is None else {
         'roc auc': roc_auc_score(actual, scores),
         'average precision': average_precision_score(actual, scores)
     }
     return util.merge_dicts(metrics, metrics_threshold)
Ejemplo n.º 24
0
    def authenticate(self, provider_id):
        if not self.is_authenticated(provider_id) and (self.username and self.password):
            self.debug('username/password set. trying to authenticate')
            self.credentials = util.merge_dicts(self.credentials, self._authenticate(provider_id))
            if self.is_authenticated(provider_id):
                self.debug('successfully authenticated')
                self._save_credentials()
            else:
                self.debug('wrong username or password')
                self.notify(32001)
        elif self.is_authenticated(provider_id):
            self.debug('already authenticated')
        else:
            self.debug('no username set to authenticate')
        self.debug(repr(self.credentials))

        return self.credentials
Ejemplo n.º 25
0
    def authenticate(self, provider_id):
        if not self.is_authenticated(provider_id) and (self.username and self.password):
            self.debug('username/password set. trying to authenticate')
            self.credentials = util.merge_dicts(self.credentials, self._authenticate(provider_id))
            if self.is_authenticated(provider_id):
                self.debug('successfully authenticated')
                self._save_credentials()
            else:
                self.debug('wrong username or password')
                self.notify(32001)
        elif self.is_authenticated(provider_id):
            self.debug('already authenticated')
        else:
            self.debug('no username set to authenticate')
        self.debug(repr(self.credentials))

        return self.credentials
Ejemplo n.º 26
0
    def __init__(self, learner, learner_kwargs, agent_kwargs, folds, data_info,
                 agent=WarmLearnerAgent, subset_prop=1, feature_subset=None, **kwargs):
        super(CrossValidationAgent, self).__init__(**kwargs)

        self.data_info = data_info

        self.child_info = []
        for train, test in folds:
            if subset_prop < 1:
                # noinspection PyUnresolvedReferences
                train = train[:int(np.floor(subset_prop * train.size))]
            self.child_info.append((agent,
                                    util.merge_dicts(agent_kwargs,
                                                     dict(learner=learner, learner_kwargs=learner_kwargs,
                                                          train_idx=train, test_idx=test,
                                                          data_info=data_info,
                                                          feature_subset=feature_subset))))

        self.score_times = []
        self.score_values = []
        self.held_out_prediction_times = []
        self.held_out_prediction_files = []
        self.valid_prediction_times = []
        self.valid_prediction_files = []
        self.test_prediction_times = []
        self.test_prediction_files = []

        self.child_score_times = dict()
        self.child_score_values = dict()
        self.child_held_out_prediction_times = dict()
        self.child_held_out_prediction_files = dict()
        self.child_held_out_idx = dict()
        self.child_valid_prediction_times = dict()
        self.child_valid_prediction_files = dict()
        self.child_test_prediction_times = dict()
        self.child_test_prediction_files = dict()

        self.communication_sleep = 0.1

        # TODO: Improve this hack!
        if agent == WarmLearnerAgent:
            self.immortal_offspring = True
        else:
            self.immortal_offspring = False
Ejemplo n.º 27
0
    def _active_learning_update_metrics(
            self, active_learner: ActiveLearner, x_dev: np.ndarray,
            y_dev: Series, stats: Stats, data_for_plotting: List[Stats],
            i: int, elapsed_train: float, elapsed_query: float,
            labeled_indices: List[int],
            semi_sup: bool) -> Tuple[Stats, List[Stats], List[int]]:
        predicted = active_learner.predict(x_dev)
        scores = None if semi_sup else active_learner.predict_proba(x_dev)[:,
                                                                           1]
        metrics = self._get_metrics(actual=y_dev,
                                    predicted=predicted,
                                    scores=scores)

        data_for_plotting.append(
            self._get_plotting_row(i, metrics, elapsed_train, elapsed_query))
        metrics = util.add_prefix_to_dict_keys(metrics, f'sample_{i+1}_')
        if i + 1 in self.active_learning_log_intervals or i == -1:
            stats = util.merge_dicts(stats, metrics)
        return stats, data_for_plotting, labeled_indices
Ejemplo n.º 28
0
def main():
    ds_names = classification_dataset_names
    models = [LogisticRegression, GradientBoostingClassifier]
    model_names = ["LogisticRegression", "GradientBoosting"]
    results = {}
    for i, n in enumerate(ds_names):
        try:
            print("Iteration: {}/{} '{}'".format(i + 1, len(ds_names), n))
            X, y = fetch_data(n, return_X_y=True)
            results = util.merge_dicts(results,
                                       compare(X, y, models,
                                               model_names))  # updates results
            pd.DataFrame(results).to_pickle('labels.pkl')
        except util.TimeoutError:
            print("Timed Out!")
    print("Done!")
    df = pd.DataFrame(results)
    df = df.rename(index=util.list_to_idx_dict(ds_names))
    df.to_pickle("labels.pkl")
Ejemplo n.º 29
0
 def _add_config_source(self, config: dict):
     self.raw = util.merge_dicts(self.raw, config)
Ejemplo n.º 30
0
}

LARGE_CAP = { # Mkt cap above $10B
    'AMD':{'key_terms':
               SEMICONDUCTOR,
           'name': 'Advanced Micro Devices'},
    'TLSA': {'key_terms':
                 ['pharmaceutical', 'cancer', 'immune disease', 'inflammation', 'therapeutics'],
             'name': 'Tiziana Life Sciences'},
    'ENVA': {'key_terms':
                 ['financial services', 'big data', 'loans', 'financing'],
             'name': 'Enova'},
    'EXR': {'key_terms':
                ['self-storage', 'Marie Kondo', 'relocating', 'moving', 'long term storage', 'warehousing'],
            'name': 'Extra Space Storage'}
}

SMALL_TIKCERS = merge_dicts((PENNY_STOCKS, SMALL_CAP))

ALL_TICKERS = merge_dicts((PENNY_STOCKS, SMALL_CAP, MID_CAP, LARGE_CAP))

NEXT_TICKERS = {
    'BWEN': {'key_terms':
                 ENERGY + MANUFACTURING,
             'name': 'Broadwind Energy'}
}
# for tick in ALL_TICKERS:
#     if tick in ['ROSE', 'RHE', 'ASPN', 'FET', 'OMI', 'IDN', 'PIRS', 'AGI', 'WPRT', 'SESN', 'RAVE', 'CGEN']:
#         continue
#     else:
#         NEXT_TICKERS[tick] = ALL_TICKERS[tick]
Ejemplo n.º 31
0
 def _apply_defaults(self, raw_dict):
     self.raw = util.merge_dicts(
         self._defaults_dict(),
         raw_dict,
     )
Ejemplo n.º 32
0
async def patch_plano_trabalho(
    cod_plano: str,
    plano_trabalho: schemas.PlanoTrabalhoUpdateSchema,
    db: Session = Depends(get_db),
    token: str = Depends(oauth2_scheme),
    user: User = Depends(fastapi_users.current_user(active=True))
    ):
    "Atualiza um plano de trabalho existente nos campos informados."
    # Validações da entrada conforme regras de negócio
    if cod_plano != plano_trabalho.cod_plano:
        raise HTTPException(
            status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail="Parâmetro cod_plano diferente do conteúdo do JSON")

    db_plano_trabalho = crud.get_plano_trabalho(db, user.cod_unidade, 
                                                        cod_plano)
    if db_plano_trabalho is None:
        raise HTTPException(
                status.HTTP_404_NOT_FOUND,
                detail="Só é possível aplicar PATCH em um recurso"+
                        " existente.")
    if db_plano_trabalho.cod_unidade != user.cod_unidade:
        raise HTTPException(
            status.HTTP_403_FORBIDDEN,
            detail="Usuário não pode alterar Plano de Trabalho"+
                    " de outra unidade.")

    # atualiza os atributos, exceto atividades
    merged_plano_trabalho = util.sa_obj_to_dict(db_plano_trabalho)
    patch_plano_trabalho = plano_trabalho.dict(exclude_unset=True)
    if patch_plano_trabalho.get("atividades", None):
        del patch_plano_trabalho["atividades"]
    merged_plano_trabalho.update(patch_plano_trabalho)

    # atualiza as atividades

    # traz a lista de atividades que está no banco
    db_atividades = util.list_to_dict(
        [
            util.sa_obj_to_dict(atividade)
            for atividade in getattr(db_plano_trabalho, "atividades", list())
        ],
        "id_atividade"
    )

    # cada atividade a ser modificada
    patch_atividades = util.list_to_dict(
        plano_trabalho.dict(exclude_unset=True).get("atividades", list()),
        "id_atividade"
    )

    merged_atividades = util.merge_dicts(db_atividades, patch_atividades)
    merged_plano_trabalho["atividades"] = util.dict_to_list(
        merged_atividades,
        "id_atividade"
    )

    # tenta validar o esquema

    # para validar o esquema, é necessário ter o atributo atividades,
    # mesmo que seja uma lista vazia
    if merged_plano_trabalho.get("atividades", None) is None:
        merged_plano_trabalho["atividades"] = []
    # valida o esquema do plano de trabalho atualizado
    try:
        merged_schema = schemas.PlanoTrabalhoSchema(**merged_plano_trabalho)
    except ValidationError as e:
        raise HTTPException(
        status.HTTP_422_UNPROCESSABLE_ENTITY,
        detail=json.loads(e.json())
    )
    
    crud.update_plano_trabalho(db, merged_schema, user.cod_unidade)
    return merged_plano_trabalho
Ejemplo n.º 33
0
 def set_args(self):
     if self.algorithm_settings.optimization_params is None:
         self.algorithm_settings.optimization_params = {}
     args = merge_dicts(self.set_stop_criterion(), self.algorithm_settings.optimization_params)
     return args
Ejemplo n.º 34
0
with tf.name_scope('inputs'):
    sents = []
    lens = []
    for i in range(4):
        sents.append(tf.placeholder(tf.int32, [BATCH_SIZE, None], name='sentences'+str(i)))
        lens.append(tf.placeholder(tf.int32, [BATCH_SIZE], name='lengths'+str(i)))
    kl_weight = tf.placeholder(tf.float32, name='kl_weight')
    generation_state = tf.placeholder(tf.float32, [BATCH_SIZE, None], name='sentence')

with tf.variable_scope('shared') as scope:
    dicts = [varec(sents[0], lens[0], embedding, generation_state)]
    scope.reuse_variables()
    for i in range(1,4):
        dicts.append(varec(sents[i], lens[i], embedding, generation_state, summary=False))
    d = merge_dicts(dicts)

with tf.name_scope('loss_overall'):
    total_loss = 0
    for i in range(4):
        weighted_loss = d["loss"+str(i)] + kl_weight * d["kld"+str(i)]
        tf.scalar_summary("weighted loss "+str(i), weighted_loss)
        total_loss += weighted_loss

    total_nll = sum(d["loss"+str(i)] for i in range(4))
    total_kld = sum(d["kld"+str(i)] for i in range(4))
    content_penalty = tf.reduce_mean(tf.square(d["content0"]-d["content1"])) +\
        tf.reduce_mean(tf.square(d["content2"]-d["content3"]))
    style_penalty = tf.reduce_mean(tf.square(d["style0"]-d["style2"])) +\
        tf.reduce_mean(tf.square(d["style1"]-d["style3"])) -\
        tf.reduce_mean(tf.abs(d["style0"]-d["style1"])) -\