Example #1
2
  def testWarnings(self):
    # Smaller than the threshold: no warning.
    c_sparse = ops.IndexedSlices(array_ops.placeholder(dtypes.float32),
                                 array_ops.placeholder(dtypes.int32),
                                 constant([4, 4, 4, 4]))
    with warnings.catch_warnings(record=True) as w:
      math_ops.mul(c_sparse, 1.0)
    self.assertEqual(0, len(w))

    # Greater than or equal to the threshold: warning.
    c_sparse = ops.IndexedSlices(array_ops.placeholder(dtypes.float32),
                                 array_ops.placeholder(dtypes.int32),
                                 constant([100, 100, 100, 100]))
    with warnings.catch_warnings(record=True) as w:
      math_ops.mul(c_sparse, 1.0)
    self.assertEqual(1, len(w))
    self.assertTrue(
        "with 100000000 elements. This may consume a large amount of memory."
        in str(w[0].message))

    # Unknown dense shape: warning.
    c_sparse = ops.IndexedSlices(array_ops.placeholder(dtypes.float32),
                                 array_ops.placeholder(dtypes.int32),
                                 array_ops.placeholder(dtypes.int32))
    with warnings.catch_warnings(record=True) as w:
      math_ops.mul(c_sparse, 1.0)
    self.assertEqual(1, len(w))
    self.assertTrue(
        "of unknown shape. This may consume a large amount of memory."
        in str(w[0].message))
Example #2
1
def test_calculate_chpi_positions():
    """Test calculation of cHPI positions
    """
    trans, rot, t = head_pos_to_trans_rot_t(read_head_pos(pos_fname))
    with warnings.catch_warnings(record=True):
        raw = Raw(chpi_fif_fname, allow_maxshield=True, preload=True)
    t -= raw.first_samp / raw.info['sfreq']
    quats = _calculate_chpi_positions(raw, verbose='debug')
    trans_est, rot_est, t_est = head_pos_to_trans_rot_t(quats)
    _compare_positions((trans, rot, t), (trans_est, rot_est, t_est), 0.003)

    # degenerate conditions
    raw_no_chpi = Raw(test_fif_fname)
    assert_raises(RuntimeError, _calculate_chpi_positions, raw_no_chpi)
    raw_bad = raw.copy()
    for d in raw_bad.info['dig']:
        if d['kind'] == FIFF.FIFFV_POINT_HPI:
            d['coord_frame'] = 999
            break
    assert_raises(RuntimeError, _calculate_chpi_positions, raw_bad)
    raw_bad = raw.copy()
    for d in raw_bad.info['dig']:
        if d['kind'] == FIFF.FIFFV_POINT_HPI:
            d['r'] = np.ones(3)
    raw_bad.crop(0, 1., copy=False)
    with warnings.catch_warnings(record=True):  # bad pos
        with catch_logging() as log_file:
            _calculate_chpi_positions(raw_bad, verbose=True)
    # ignore HPI info header and [done] footer
    for line in log_file.getvalue().strip().split('\n')[4:-1]:
        assert_true('0/5 good' in line)
Example #3
1
    def test_long_cache_keys_shortened(self):
        cache_settings = {
            'default': {
                'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
                'LOCATION': os.path.join(TOP_DIR, 'test.cache'),
            }
        }
        long_key_string = "X" * 251

        with override_settings(CACHES=cache_settings):
            with warnings.catch_warnings(record=True) as w:
                warnings.simplefilter("always")
                # memcached limits key length to 250
                cache.set(long_key_string, "hello cached world")

                self.assertEqual(len(w), 1)
                self.assertIsInstance(w[0].message, CacheKeyWarning)

        # Activate optional cache key length checker
        cache_settings['default']['KEY_FUNCTION'] = 'mainsite.utils.filter_cache_key'

        with override_settings(CACHES=cache_settings):
            with warnings.catch_warnings(record=True) as w:
                warnings.simplefilter("always")
                # memcached limits key length to 250
                cache.set(long_key_string, "hello cached world")

                self.assertEqual(len(w), 0)

                retrieved = cache.get(long_key_string)

                self.assertEqual(retrieved, "hello cached world")
Example #4
0
    def test_iloc_getitem_multiindex(self):
        mi_labels = DataFrame(np.random.randn(4, 3),
                              columns=[['i', 'i', 'j'], ['A', 'A', 'B']],
                              index=[['i', 'i', 'j', 'k'],
                                     ['X', 'X', 'Y', 'Y']])

        mi_int = DataFrame(np.random.randn(3, 3),
                           columns=[[2, 2, 4], [6, 8, 10]],
                           index=[[4, 4, 8], [8, 10, 12]])

        # the first row
        rs = mi_int.iloc[0]
        with catch_warnings(record=True):
            xp = mi_int.ix[4].ix[8]
        tm.assert_series_equal(rs, xp, check_names=False)
        assert rs.name == (4, 8)
        assert xp.name == 8

        # 2nd (last) columns
        rs = mi_int.iloc[:, 2]
        with catch_warnings(record=True):
            xp = mi_int.ix[:, 2]
        tm.assert_series_equal(rs, xp)

        # corner column
        rs = mi_int.iloc[2, 2]
        with catch_warnings(record=True):
            xp = mi_int.ix[:, 2].ix[2]
        assert rs == xp

        # this is basically regular indexing
        rs = mi_labels.iloc[2, 2]
        with catch_warnings(record=True):
            xp = mi_labels.ix['j'].ix[:, 'j'].ix[0, 0]
        assert rs == xp
def test_deprecated_score_func():
    # test that old deprecated way of passing a score / loss function is still
    # supported
    X, y = make_classification(n_samples=200, n_features=100, random_state=0)
    clf = LinearSVC(random_state=0)
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]}, scoring="f1")
    cv.fit(X[:180], y[:180])
    y_pred = cv.predict(X[180:])
    C = cv.best_estimator_.C

    clf = LinearSVC(random_state=0)
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]}, score_func=f1_score)
    with warnings.catch_warnings(record=True):
        # catch deprecation warning
        cv.fit(X[:180], y[:180])
    y_pred_func = cv.predict(X[180:])
    C_func = cv.best_estimator_.C

    assert_array_equal(y_pred, y_pred_func)
    assert_equal(C, C_func)

    # test loss where greater is worse
    def f1_loss(y_true_, y_pred_):
        return -f1_score(y_true_, y_pred_)

    clf = LinearSVC(random_state=0)
    cv = GridSearchCV(clf, {'C': [0.1, 1.0]}, loss_func=f1_loss)
    with warnings.catch_warnings(record=True):
        # catch deprecation warning
        cv.fit(X[:180], y[:180])
    y_pred_loss = cv.predict(X[180:])
    C_loss = cv.best_estimator_.C

    assert_array_equal(y_pred, y_pred_loss)
    assert_equal(C, C_loss)
Example #6
0
 def test_deprecated_callbacks(self):
     # Tests that callback functions that return values are still supported but that warnings are generated
     
     def returns_cube(cube, field, filename):
         return cube
         
     def returns_no_cube(cube, field, filename):
         return iris.io.NO_CUBE
         
     fname = tests.get_data_path(["PP", "trui", "air_temp_init", "200812011200__qwqu12ff.initanl.pp"])
     
     # Catch all warnings for returns_cube
     with warnings.catch_warnings(record=True) as generated_warnings_cube:
         warnings.simplefilter("always")
         r = iris.load(fname, callback=returns_cube)
         
         # Test that our warnings are present in the generated warnings:
         gen_warnings_cube = [str(x.message) for x in generated_warnings_cube]
         self.assertIn(iris.io.CALLBACK_DEPRECATION_MSG, gen_warnings_cube, "Callback deprecation warning message not issued.")
     
     # Catch all warnings for returns_no_cube
     with warnings.catch_warnings(record=True) as generated_warnings_no_cube:
         warnings.simplefilter("always")  
         r = iris.load(fname, callback=returns_no_cube)
         
         # Test that our warnings are present in the generated warnings:
         gen_warnings_no_cube = [str(x.message) for x in generated_warnings_no_cube]
         self.assertIn(iris.io.CALLBACK_DEPRECATION_MSG, gen_warnings_no_cube, "Callback deprecation warning message not issued.")
Example #7
0
def test_sparse_randomized_pca_inverse():
    """Test that RandomizedPCA is inversible on sparse data"""
    rng = np.random.RandomState(0)
    n, p = 50, 3
    X = rng.randn(n, p)  # spherical data
    X[:, 1] *= 0.00001  # make middle component relatively small
    # no large means because the sparse version of randomized pca does not do
    # centering to avoid breaking the sparsity
    X = csr_matrix(X)

    # same check that we can find the original data from the transformed signal
    # (since the data is almost of rank n_components)
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter("always", DeprecationWarning)
        pca = RandomizedPCA(n_components=2, random_state=0).fit(X)
        assert_equal(len(w), 1)
        assert_equal(w[0].category, DeprecationWarning)

    Y = pca.transform(X)

    Y_inverse = pca.inverse_transform(Y)
    assert_almost_equal(X.todense(), Y_inverse, decimal=2)

    # same as above with whitening (approximate reconstruction)
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter("always", DeprecationWarning)
        pca = RandomizedPCA(n_components=2, whiten=True, random_state=0).fit(X)
        assert_equal(len(w), 1)
        assert_equal(w[0].category, DeprecationWarning)

    Y = pca.transform(X)
    Y_inverse = pca.inverse_transform(Y)
    relative_max_delta = (np.abs(X.todense() - Y_inverse) / np.abs(X).mean()).max()
    # XXX: this does not seam to work as expected:
    assert_almost_equal(relative_max_delta, 0.91, decimal=2)
Example #8
0
        def run_tests(df, rhs, right):
            # label, index, slice
            r, i, s = list('bcd'), [1, 2, 3], slice(1, 4)
            c, j, l = ['joe', 'jolie'], [1, 2], slice(1, 3)

            left = df.copy()
            left.loc[r, c] = rhs
            tm.assert_frame_equal(left, right)

            left = df.copy()
            left.iloc[i, j] = rhs
            tm.assert_frame_equal(left, right)

            left = df.copy()
            with catch_warnings(record=True):
                left.ix[s, l] = rhs
            tm.assert_frame_equal(left, right)

            left = df.copy()
            with catch_warnings(record=True):
                left.ix[i, j] = rhs
            tm.assert_frame_equal(left, right)

            left = df.copy()
            with catch_warnings(record=True):
                left.ix[r, c] = rhs
            tm.assert_frame_equal(left, right)
def check_classifiers_input_shapes(name, Classifier):
    iris = load_iris()
    X, y = iris.data, iris.target
    X, y = shuffle(X, y, random_state=1)
    X = StandardScaler().fit_transform(X)
    # catch deprecation warnings
    with warnings.catch_warnings(record=True):
        classifier = Classifier()
    set_fast_parameters(classifier)
    set_random_state(classifier)
    # fit
    classifier.fit(X, y)
    y_pred = classifier.predict(X)

    set_random_state(classifier)
    # Check that when a 2D y is given, a DataConversionWarning is
    # raised
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter("always", DataConversionWarning)
        warnings.simplefilter("ignore", RuntimeWarning)
        classifier.fit(X, y[:, np.newaxis])
    msg = "expected 1 DataConversionWarning, got: %s" % (
        ", ".join([str(w_x) for w_x in w]))
    assert_equal(len(w), 1, msg)
    assert_array_equal(y_pred, classifier.predict(X))
Example #10
0
    def test_get_url_shortener(self):
        us_settings.URL_SHORTENER_BACKEND = 'mymodule.myclass'
        try:
            with warnings.catch_warnings(record=True) as w:
                self.assertEquals(get_url_shortener(), default_backend)
                self.assertTrue(issubclass(w[-1].metatype, RuntimeWarning))
                self.assertEquals(
                    str(w[-1].message),
                    'mymodule.myclass backend cannot be imported')
        except AttributeError:
            # Fail under Python2.5, because of'warnings.catch_warnings'
            pass

        us_settings.URL_SHORTENER_BACKEND = 'gstudio.tests.custom_url_shortener'
        try:
            with warnings.catch_warnings(record=True) as w:
                self.assertEquals(get_url_shortener(), default_backend)
                self.assertTrue(issubclass(w[-1].metatype, RuntimeWarning))
                self.assertEquals(
                    str(w[-1].message),
                    'This backend only exists for testing')
        except AttributeError:
            # Fail under Python2.5, because of'warnings.catch_warnings'
            pass

        us_settings.URL_SHORTENER_BACKEND = 'gstudio.url_shortener'\
                                            '.backends.default'
        self.assertEquals(get_url_shortener(), default_backend)
Example #11
0
 def setUp(self):
     self.aln_file = [TEST_ALIGN_FILE1,
                      TEST_ALIGN_FILE2,
                      TEST_ALIGN_FILE3,
                      TEST_ALIGN_FILE4,
                      TEST_ALIGN_FILE5,
                      TEST_ALIGN_FILE6]
     alns = []
     for i in self.aln_file:
         if i[1] == 'parse':
             nucl = SeqIO.parse(i[0][0], 'fasta', alphabet=IUPAC.IUPACUnambiguousDNA())
             prot = AlignIO.read(i[0][1], 'clustal', alphabet=IUPAC.protein)
             with warnings.catch_warnings():
                 warnings.simplefilter('ignore')
                 caln = codonalign.build(prot, nucl, alphabet=codonalign.default_codon_alphabet)
         elif i[1] == 'index':
             nucl = SeqIO.index(i[0][0], 'fasta', alphabet=IUPAC.IUPACUnambiguousDNA())
             prot = AlignIO.read(i[0][1], 'clustal', alphabet=IUPAC.protein)
             with warnings.catch_warnings():
                 warnings.simplefilter('ignore')
                 caln = codonalign.build(prot, nucl, alphabet=codonalign.default_codon_alphabet, max_score=20)
         elif i[1] == 'id':
             nucl = SeqIO.parse(i[0][0], 'fasta', alphabet=IUPAC.IUPACUnambiguousDNA())
             prot = AlignIO.read(i[0][1], 'clustal', alphabet=IUPAC.protein)
             with open(i[0][2]) as handle:
                 id = dict((i.split()[0], i.split()[1]) for i in handle)
             with warnings.catch_warnings():
                 warnings.simplefilter('ignore')
                 caln = codonalign.build(prot, nucl, corr_dict=id, alphabet=codonalign.default_codon_alphabet)
         alns.append(caln)
         nucl.close()  # Close the indexed FASTA file
     self.alns = alns
def check_clustering(name, Alg):
    X, y = make_blobs(n_samples=50, random_state=1)
    X, y = shuffle(X, y, random_state=7)
    X = StandardScaler().fit_transform(X)
    n_samples, n_features = X.shape
    # catch deprecation and neighbors warnings
    with warnings.catch_warnings(record=True):
        alg = Alg()
    set_fast_parameters(alg)
    if hasattr(alg, "n_clusters"):
        alg.set_params(n_clusters=3)
    set_random_state(alg)
    if name == 'AffinityPropagation':
        alg.set_params(preference=-100)
        alg.set_params(max_iter=100)

    # fit
    alg.fit(X)
    # with lists
    alg.fit(X.tolist())

    assert_equal(alg.labels_.shape, (n_samples,))
    pred = alg.labels_
    assert_greater(adjusted_rand_score(pred, y), 0.4)
    # fit another time with ``fit_predict`` and compare results
    if name is 'SpectralClustering':
        # there is no way to make Spectral clustering deterministic :(
        return
    set_random_state(alg)
    with warnings.catch_warnings(record=True):
        pred2 = alg.fit_predict(X)
    assert_array_equal(pred, pred2)
Example #13
0
def test_all():
    """Test maxwell filter using all options"""
    raw_fnames = (raw_fname, raw_fname, erm_fname, sample_fname)
    sss_fnames = (sss_st1FineCalCrossTalkRegIn_fname,
                  sss_st1FineCalCrossTalkRegInTransSample_fname,
                  sss_erm_st1FineCalCrossTalkRegIn_fname,
                  sss_samp_fname)
    fine_cals = (fine_cal_fname,
                 fine_cal_fname,
                 fine_cal_fname,
                 fine_cal_mgh_fname)
    coord_frames = ('head', 'head', 'meg', 'head')
    ctcs = (ctc_fname, ctc_fname, ctc_fname, ctc_mgh_fname)
    mins = (3.5, 3.5, 1.2, 0.9)
    meds = (10.9, 10.4, 3.2, 6.)
    st_durs = (1., 1., 1., None)
    destinations = (None, sample_fname, None, None)
    origins = (mf_head_origin,
               mf_head_origin,
               mf_meg_origin,
               mf_head_origin)
    for ii, rf in enumerate(raw_fnames):
        with warnings.catch_warnings(record=True):  # maxshield
            raw = Raw(rf, allow_maxshield=True).crop(0., 1., copy=False)
        with warnings.catch_warnings(record=True):  # head fit off-center
            sss_py = maxwell_filter(
                raw, calibration=fine_cals[ii], cross_talk=ctcs[ii],
                st_duration=st_durs[ii], coord_frame=coord_frames[ii],
                destination=destinations[ii], origin=origins[ii])
        sss_mf = Raw(sss_fnames[ii])
        assert_meg_snr(sss_py, sss_mf, mins[ii], meds[ii], msg=rf)
Example #14
0
    def test_read_chunks_115(self):
        files_115 = [self.dta2_115, self.dta3_115, self.dta4_115,
                     self.dta14_115, self.dta15_115, self.dta16_115,
                     self.dta17_115, self.dta18_115, self.dta19_115,
                     self.dta20_115]

        for fname in files_115:
            for chunksize in 1,2:
                for convert_categoricals in False, True:
                    for convert_dates in False, True:

                        # Read the whole file
                        with warnings.catch_warnings(record=True) as w:
                            warnings.simplefilter("always")
                            parsed = read_stata(fname, convert_categoricals=convert_categoricals,
                                                convert_dates=convert_dates)

                        # Compare to what we get when reading by chunk
                        itr = read_stata(fname, iterator=True, convert_dates=convert_dates,
                                         convert_categoricals=convert_categoricals)
                        pos = 0
                        for j in range(5):
                            with warnings.catch_warnings(record=True) as w:
                                warnings.simplefilter("always")
                                try:
                                    chunk = itr.read(chunksize)
                                except StopIteration:
                                    break
                            from_frame = parsed.iloc[pos:pos+chunksize, :]
                            tm.assert_frame_equal(from_frame,
                                                  chunk,
                                                  check_dtype=False,
                                                  check_datetimelike_compat=True)

                            pos += chunksize
Example #15
0
        def run_tests(df, rhs, right):
            # label, index, slice
            lbl_one, idx_one, slice_one = list('bcd'), [1, 2, 3], slice(1, 4)
            lbl_two, idx_two, slice_two = ['joe', 'jolie'], [1, 2], slice(1, 3)

            left = df.copy()
            left.loc[lbl_one, lbl_two] = rhs
            tm.assert_frame_equal(left, right)

            left = df.copy()
            left.iloc[idx_one, idx_two] = rhs
            tm.assert_frame_equal(left, right)

            left = df.copy()
            with catch_warnings(record=True):
                # XXX: finer-filter here.
                simplefilter("ignore")
                left.ix[slice_one, slice_two] = rhs
            tm.assert_frame_equal(left, right)

            left = df.copy()
            with catch_warnings(record=True):
                simplefilter("ignore")
                left.ix[idx_one, idx_two] = rhs
            tm.assert_frame_equal(left, right)

            left = df.copy()
            with catch_warnings(record=True):
                simplefilter("ignore")
                left.ix[lbl_one, lbl_two] = rhs
            tm.assert_frame_equal(left, right)
Example #16
0
def test_head_translation():
    """Test Maxwell filter head translation"""
    with warnings.catch_warnings(record=True):  # maxshield
        raw = Raw(raw_fname, allow_maxshield=True).crop(0., 1., False)
    # First try with an unchanged destination
    raw_sss = maxwell_filter(raw, destination=raw_fname,
                             origin=mf_head_origin, regularize=None,
                             bad_condition='ignore')
    assert_meg_snr(raw_sss, Raw(sss_std_fname).crop(0., 1., False), 200.)
    # Now with default
    with warnings.catch_warnings(record=True):
        with catch_logging() as log:
            raw_sss = maxwell_filter(raw, destination=mf_head_origin,
                                     origin=mf_head_origin, regularize=None,
                                     bad_condition='ignore', verbose='warning')
    assert_true('over 25 mm' in log.getvalue())
    assert_meg_snr(raw_sss, Raw(sss_trans_default_fname), 125.)
    destination = np.eye(4)
    destination[2, 3] = 0.04
    assert_allclose(raw_sss.info['dev_head_t']['trans'], destination)
    # Now to sample's head pos
    with warnings.catch_warnings(record=True):
        with catch_logging() as log:
            raw_sss = maxwell_filter(raw, destination=sample_fname,
                                     origin=mf_head_origin, regularize=None,
                                     bad_condition='ignore', verbose='warning')
    assert_true('= 25.6 mm' in log.getvalue())
    assert_meg_snr(raw_sss, Raw(sss_trans_sample_fname), 350.)
    assert_allclose(raw_sss.info['dev_head_t']['trans'],
                    read_info(sample_fname)['dev_head_t']['trans'])
    # Degenerate cases
    assert_raises(RuntimeError, maxwell_filter, raw,
                  destination=mf_head_origin, coord_frame='meg')
    assert_raises(ValueError, maxwell_filter, raw, destination=[0.] * 4)
Example #17
0
    def test_random_pair(self) :
        self.assertRaises(ValueError, dedupe.core.randomPairs, 1, 10)
        assert dedupe.core.randomPairs(10, 10).any()
        random.seed(123)
        numpy.random.seed(123)
        random_pairs = dedupe.core.randomPairs(10, 5)
        assert numpy.array_equal(random_pairs, 
                                 numpy.array([[ 0,  3],
                                              [ 3,  8],
                                              [ 4,  9],
                                              [ 5,  9],
                                              [ 2,  3]]))

        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter("always")
            dedupe.core.randomPairs(10, 10**6)
            assert len(w) == 1
            assert str(w[-1].message) == "Requested sample of size 1000000, only returning 45 possible pairs"

        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter("always")
            dedupe.core.randomPairs(10**40, 10)
            assert len(w) == 2
            assert str(w[0].message) == "There may be duplicates in the sample"
            assert "Asked to sample pairs from" in str(w[1].message)

        random.seed(123)
        numpy.random.seed(123)
        assert numpy.array_equal(dedupe.core.randomPairs(11**9, 1),
                                 numpy.array([[1228959102, 1840268610]]))
Example #18
0
def test_make_morph_maps():
    """Test reading and creating morph maps."""
    # make a new fake subjects_dir
    tempdir = _TempDir()
    for subject in ('sample', 'sample_ds', 'fsaverage_ds'):
        os.mkdir(op.join(tempdir, subject))
        os.mkdir(op.join(tempdir, subject, 'surf'))
        regs = ('reg', 'left_right') if subject == 'fsaverage_ds' else ('reg',)
        for hemi in ['lh', 'rh']:
            for reg in regs:
                args = [subject, 'surf', hemi + '.sphere.' + reg]
                copyfile(op.join(subjects_dir, *args),
                         op.join(tempdir, *args))

    for subject_from, subject_to, xhemi in (
            ('fsaverage_ds', 'sample_ds', False),
            ('fsaverage_ds', 'fsaverage_ds', True)):
        # trigger the creation of morph-maps dir and create the map
        with warnings.catch_warnings(record=True):
            mmap = read_morph_map(subject_from, subject_to, tempdir,
                                  xhemi=xhemi)
        mmap2 = read_morph_map(subject_from, subject_to, subjects_dir,
                               xhemi=xhemi)
        assert_equal(len(mmap), len(mmap2))
        for m1, m2 in zip(mmap, mmap2):
            # deal with sparse matrix stuff
            diff = (m1 - m2).data
            assert_allclose(diff, np.zeros_like(diff), atol=1e-3, rtol=0)

    # This will also trigger creation, but it's trivial
    with warnings.catch_warnings(record=True):
        mmap = read_morph_map('sample', 'sample', subjects_dir=tempdir)
    for mm in mmap:
        assert_true((mm - sparse.eye(mm.shape[0], mm.shape[0])).sum() == 0)
    def test_array_richcompare_legacy_weirdness(self):
        # It doesn't really work to use assert_deprecated here, b/c part of
        # the point of assert_deprecated is to check that when warnings are
        # set to "error" mode then the error is propagated -- which is good!
        # But here we are testing a bunch of code that is deprecated *because*
        # it has the habit of swallowing up errors and converting them into
        # different warnings. So assert_warns will have to be sufficient.
        assert_warns(FutureWarning, lambda: np.arange(2) == "a")
        assert_warns(FutureWarning, lambda: np.arange(2) != "a")
        # No warning for scalar comparisons
        with warnings.catch_warnings():
            warnings.filterwarnings("error")
            assert_(not (np.array(0) == "a"))
            assert_(np.array(0) != "a")
            assert_(not (np.int16(0) == "a"))
            assert_(np.int16(0) != "a")

        for arg1 in [np.asarray(0), np.int16(0)]:
            struct = np.zeros(2, dtype="i4,i4")
            for arg2 in [struct, "a"]:
                for f in [operator.lt, operator.le, operator.gt, operator.ge]:
                    if sys.version_info[0] >= 3:
                        # py3
                        with warnings.catch_warnings() as l:
                            warnings.filterwarnings("always")
                            assert_raises(TypeError, f, arg1, arg2)
                            assert_(not l)
                    else:
                        # py2
                        assert_warns(DeprecationWarning, f, arg1, arg2)
Example #20
0
    def test_random_pair_match(self) :
        self.assertRaises(ValueError, dedupe.core.randomPairsMatch, 1, 0, 10)
        self.assertRaises(ValueError, dedupe.core.randomPairsMatch, 0, 0, 10)
        self.assertRaises(ValueError, dedupe.core.randomPairsMatch, 0, 1, 10)

        assert len(dedupe.core.randomPairsMatch(100, 100, 100)) == 100
        assert len(dedupe.core.randomPairsMatch(10, 10, 99)) == 99


        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter("always")
            pairs = dedupe.core.randomPairsMatch(10, 10, 200)
            assert len(w) == 1
            assert str(w[-1].message) == "Requested sample of size 200, only returning 100 possible pairs"

        assert len(pairs) == 100

        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter("always")
            pairs = dedupe.core.randomPairsMatch(10, 10, 200)
            assert len(w) == 1
            assert str(w[-1].message) == "Requested sample of size 200, only returning 100 possible pairs"


        random.seed(123)
        numpy.random.seed(123)
        pairs = dedupe.core.randomPairsMatch(10, 10, 10)
        assert pairs == set([(7, 3), (3, 3), (2, 9), (6, 0), (2, 0), 
                             (1, 9), (9, 4), (0, 4), (1, 0), (1, 1)])
Example #21
0
def test_misspecifications():
    # Tests for model specification and misspecification exceptions
    endog = np.arange(20).reshape(10,2)

    # Bad trend specification
    assert_raises(ValueError, varmax.VARMAX, endog, order=(1,0), trend='')

    # Bad error_cov_type specification
    assert_raises(ValueError, varmax.VARMAX, endog, order=(1,0), error_cov_type='')

    # Bad order specification
    assert_raises(ValueError, varmax.VARMAX, endog, order=(0,0))

    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter('always')
        varmax.VARMAX(endog, order=(1,1))

    # Warning with VARMA specification
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter('always')

        varmax.VARMAX(endog, order=(1,1))

        message = ('Estimation of VARMA(p,q) models is not generically robust,'
                   ' due especially to identification issues.')
        assert_equal(str(w[0].message), message)
    warnings.resetwarnings()
Example #22
0
def test_class_weight_classifiers():
    # test that class_weight works and that the semantics are consistent
    classifiers = all_estimators(type_filter="classifier")

    with warnings.catch_warnings(record=True):
        classifiers = [c for c in classifiers if "class_weight" in c[1]().get_params().keys()]

    for n_centers in [2, 3]:
        # create a very noisy dataset
        X, y = make_blobs(centers=n_centers, random_state=0, cluster_std=20)
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0)
        for name, Classifier in classifiers:
            if name == "NuSVC":
                # the sparse version has a parameter that doesn't do anything
                continue
            if name.endswith("NB"):
                # NaiveBayes classifiers have a somewhat different interface.
                # FIXME SOON!
                continue
            if n_centers == 2:
                class_weight = {0: 1000, 1: 0.0001}
            else:
                class_weight = {0: 1000, 1: 0.0001, 2: 0.0001}

            with warnings.catch_warnings(record=True):
                classifier = Classifier(class_weight=class_weight)
            if hasattr(classifier, "n_iter"):
                classifier.set_params(n_iter=100)

            set_random_state(classifier)
            classifier.fit(X_train, y_train)
            y_pred = classifier.predict(X_test)
            assert_greater(np.mean(y_pred == 0), 0.9)
Example #23
0
def _check_predict_proba(clf, X, y):
    proba = clf.predict_proba(X)
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        # We know that we can have division by zero
        log_proba = clf.predict_log_proba(X)

    y = np.atleast_1d(y)
    if y.ndim == 1:
        y = np.reshape(y, (-1, 1))

    n_outputs = y.shape[1]
    n_samples = len(X)

    if n_outputs == 1:
        proba = [proba]
        log_proba = [log_proba]

    for k in xrange(n_outputs):
        assert_equal(proba[k].shape[0], n_samples)
        assert_equal(proba[k].shape[1], len(np.unique(y[:, k])))
        assert_array_equal(proba[k].sum(axis=1), np.ones(len(X)))
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            # We know that we can have division by zero
            assert_array_equal(np.log(proba[k]), log_proba[k])
Example #24
0
    def test_popen(self):
        mswindows = (sys.platform == "win32")

        if mswindows:
            command = '"{}" -c "print(\'Hello\')"'.format(sys.executable)
        else:
            command = "'{}' -c 'print(\"Hello\")'".format(sys.executable)
        with warnings.catch_warnings():
            warnings.simplefilter("ignore", DeprecationWarning)
            with platform.popen(command) as stdout:
                hello = stdout.read().strip()
                stdout.close()
                self.assertEqual(hello, "Hello")

        data = 'plop'
        if mswindows:
            command = '"{}" -c "import sys; data=sys.stdin.read(); exit(len(data))"'
        else:
            command = "'{}' -c 'import sys; data=sys.stdin.read(); exit(len(data))'"
        command = command.format(sys.executable)
        with warnings.catch_warnings():
            warnings.simplefilter("ignore", DeprecationWarning)
            with platform.popen(command, 'w') as stdin:
                stdout = stdin.write(data)
                ret = stdin.close()
                self.assertIsNotNone(ret)
                if os.name == 'nt':
                    returncode = ret
                else:
                    returncode = ret >> 8
                self.assertEqual(returncode, len(data))
Example #25
0
    def test_graph_iterative(self):
        graph = MigrationGraph()
        root = ("app_a", "1")
        graph.add_node(root, None)
        expected = [root]
        for i in range(2, 1000):
            parent = ("app_a", str(i - 1))
            child = ("app_a", str(i))
            graph.add_node(child, None)
            graph.add_dependency(str(i), child, parent)
            expected.append(child)
        leaf = expected[-1]

        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always', RuntimeWarning)
            forwards_plan = graph.forwards_plan(leaf)

        self.assertEqual(len(w), 1)
        self.assertTrue(issubclass(w[-1].category, RuntimeWarning))
        self.assertEqual(str(w[-1].message), RECURSION_DEPTH_WARNING)
        self.assertEqual(expected, forwards_plan)

        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always', RuntimeWarning)
            backwards_plan = graph.backwards_plan(root)

        self.assertEqual(len(w), 1)
        self.assertTrue(issubclass(w[-1].category, RuntimeWarning))
        self.assertEqual(str(w[-1].message), RECURSION_DEPTH_WARNING)
        self.assertEqual(expected[::-1], backwards_plan)
Example #26
0
    def _check_roundtrip(self, frame):
        _skip_if_no_MySQLdb()
        drop_sql = "DROP TABLE IF EXISTS test_table"
        cur = self.db.cursor()
        with warnings.catch_warnings():
            warnings.filterwarnings("ignore", "Unknown table.*")
            cur.execute(drop_sql)
        sql.write_frame(frame, name='test_table', con=self.db, flavor='mysql')
        result = sql.read_frame("select * from test_table", self.db)

        # HACK! Change this once indexes are handled properly.
        result.index = frame.index
        result.index.name = frame.index.name

        expected = frame
        tm.assert_frame_equal(result, expected)

        frame['txt'] = ['a'] * len(frame)
        frame2 = frame.copy()
        index = Index(lrange(len(frame2))) + 10
        frame2['Idx'] = index
        drop_sql = "DROP TABLE IF EXISTS test_table2"
        cur = self.db.cursor()
        with warnings.catch_warnings():
            warnings.filterwarnings("ignore", "Unknown table.*")
            cur.execute(drop_sql)
        sql.write_frame(frame2, name='test_table2', con=self.db, flavor='mysql')
        result = sql.read_frame("select * from test_table2", self.db,
                                index_col='Idx')
        expected = frame.copy()

        # HACK! Change this once indexes are handled properly.
        expected.index = index
        expected.index.names = result.index.names
        tm.assert_frame_equal(expected, result)
Example #27
0
def test_evoked_io_from_epochs():
    """Test IO of evoked data made from epochs
    """
    # offset our tmin so we don't get exactly a zero value when decimating
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter("always")
        epochs = Epochs(raw, events[:4], event_id, tmin + 0.011, tmax, picks=picks, baseline=(None, 0), decim=5)
    assert_true(len(w) == 1)
    evoked = epochs.average()
    evoked.save(op.join(tempdir, "evoked-ave.fif"))
    evoked2 = read_evokeds(op.join(tempdir, "evoked-ave.fif"))[0]
    assert_allclose(evoked.data, evoked2.data, rtol=1e-4, atol=1e-20)
    assert_allclose(evoked.times, evoked2.times, rtol=1e-4, atol=1 / evoked.info["sfreq"])

    # now let's do one with negative time
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter("always")
        epochs = Epochs(raw, events[:4], event_id, 0.1, tmax, picks=picks, baseline=(0.1, 0.2), decim=5)
    evoked = epochs.average()
    evoked.save(op.join(tempdir, "evoked-ave.fif"))
    evoked2 = read_evokeds(op.join(tempdir, "evoked-ave.fif"))[0]
    assert_allclose(evoked.data, evoked2.data, rtol=1e-4, atol=1e-20)
    assert_allclose(evoked.times, evoked2.times, rtol=1e-4, atol=1e-20)

    # should be equivalent to a cropped original
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter("always")
        epochs = Epochs(raw, events[:4], event_id, -0.2, tmax, picks=picks, baseline=(0.1, 0.2), decim=5)
    evoked = epochs.average()
    evoked.crop(0.099, None)
    assert_allclose(evoked.data, evoked2.data, rtol=1e-4, atol=1e-20)
    assert_allclose(evoked.times, evoked2.times, rtol=1e-4, atol=1e-20)
Example #28
0
def test_read_epochs_bad_events():
    """Test epochs when events are at the beginning or the end of the file
    """
    # Event at the beginning
    epochs = Epochs(
        raw, np.array([[raw.first_samp, 0, event_id]]), event_id, tmin, tmax, picks=picks, baseline=(None, 0)
    )
    with warnings.catch_warnings(record=True):
        evoked = epochs.average()

    epochs = Epochs(
        raw, np.array([[raw.first_samp, 0, event_id]]), event_id, tmin, tmax, picks=picks, baseline=(None, 0)
    )
    epochs.drop_bad_epochs()
    with warnings.catch_warnings(record=True):
        evoked = epochs.average()

    # Event at the end
    epochs = Epochs(
        raw, np.array([[raw.last_samp, 0, event_id]]), event_id, tmin, tmax, picks=picks, baseline=(None, 0)
    )

    with warnings.catch_warnings(record=True):
        evoked = epochs.average()
        assert evoked
    warnings.resetwarnings()
Example #29
0
def read_rgb_f32(path_to_sequence: str) -> pims.FramesSequence:
    with warnings.catch_warnings():
        warnings.simplefilter('ignore')
        return _to_float32(pims.open(path_to_sequence))
Example #30
0
    def _runCompatible(self, compatibleList):
        """Calculate the metric values for set of (parent and child) bundles, as well as the summary stats,
        and write to disk.

        Parameters
        -----------
        compatibleList : list
            List of dictionary keys, of the metricBundles which can be calculated together.
            This means they are 'compatible' and have the same slicer, constraint, and non-conflicting
            mappers and stackers.
        """
        if self.verbose:
            print('Running metrics %s' % compatibleList)

        bDict = self.bundleDict  #  {key: self.bundleDict.get(key) for key in compatibleList}

        # Find the unique stackers and maps. These are already "compatible" (as id'd by compatibleList).
        uniqStackers = []
        allStackers = []
        uniqMaps = []
        allMaps = []
        for b in bDict.values():
            allStackers += b.stackerList
            allMaps += b.mapsList
        for s in allStackers:
            if s not in uniqStackers:
                uniqStackers.append(s)
        for m in allMaps:
            if m not in uniqMaps:
                uniqMaps.append(m)

        if len(uniqMaps) > 0:
            print("Got some maps .. that was unexpected at the moment. Can't use them here yet.")

        # Set up all of the metric values, including for the child bundles.
        for k in compatibleList:
            b = self.bundleDict[k]
            b._setupMetricValues()
            for cb in b.childBundles.values():
                cb._setupMetricValues()
        # Calculate the metric values.
        for i, slicePoint in enumerate(self.slicer):
            ssoObs = slicePoint['obs']
            for j, Hval in enumerate(slicePoint['Hvals']):
                # Run stackers to add extra columns (that depend on Hval)
                with warnings.catch_warnings():
                    warnings.simplefilter('ignore')
                    for s in uniqStackers:
                        ssoObs = s.run(ssoObs, slicePoint['orbit']['H'], Hval)
                # Run all the parent metrics.
                for k in compatibleList:
                    b = self.bundleDict[k]
                    # Mask the parent metric (and then child metrics) if there was no data.
                    if len(ssoObs) == 0:
                        b.metricValues.mask[i][j] = True
                        for cb in list(b.childBundles.values()):
                            cb.metricValues.mask[i][j] = True
                    # Otherwise, calculate the metric value for the parent, and then child.
                    else:
                        # Calculate for the parent.
                        mVal = b.metric.run(ssoObs, slicePoint['orbit'], Hval)
                        # Mask if the parent metric returned a bad value.
                        if mVal == b.metric.badval:
                            b.metricValues.mask[i][j] = True
                            for cb in b.childBundles.values():
                                cb.metricValues.mask[i][j] = True
                        # Otherwise, set the parent value and calculate the child metric values as well.
                        else:
                            b.metricValues.data[i][j] = mVal
                            for cb in b.childBundles.values():
                                childVal = cb.metric.run(ssoObs, slicePoint['orbit'], Hval, mVal)
                                if childVal == cb.metric.badval:
                                    cb.metricValues.mask[i][j] = True
                                else:
                                    cb.metricValues.data[i][j] = childVal
        for k in compatibleList:
            b = self.bundleDict[k]
            b.computeSummaryStats(self.resultsDb)
            for cB in b.childBundles.values():
                cB.computeSummaryStats(self.resultsDb)
                # Write to disk.
                cB.write(outDir=self.outDir, resultsDb=self.resultsDb)
            # Write to disk.
            b.write(outDir=self.outDir, resultsDb=self.resultsDb)
Example #31
0
def soft_thresholding_vector(a, lamda):
    """Soft-thresholding for vectors."""
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        return np.maximum(1 - lamda / np.linalg.norm(a), 0) * a
Example #32
0
def main(night_name=None, files=None):
    # ----------------------------------------------------------------------
    # Set up
    # ----------------------------------------------------------------------
    # get parameters from config files/run time args/load paths + calibdb
    p = spirouStartup.Begin(recipe=__NAME__)
    p = spirouStartup.LoadArguments(p,
                                    night_name,
                                    files,
                                    mainfitsdir='reduced')
    p = spirouStartup.InitialFileSetup(p, calibdb=True)
    # set up function name
    main_name = __NAME__ + '.main()'

    # ----------------------------------------------------------------------
    # Load first file
    # ----------------------------------------------------------------------
    loc = ParamDict()
    rd = spirouImage.ReadImage(p, p['FITSFILENAME'])
    loc['DATA'], loc['DATAHDR'], loc['XDIM'], loc['YDIM'] = rd
    loc.set_sources(['DATA', 'DATAHDR', 'XDIM', 'YDIM'], main_name)

    # ----------------------------------------------------------------------
    # Get object name, airmass and berv
    # ----------------------------------------------------------------------
    # Get object name
    loc['OBJNAME'] = spirouImage.GetObjName(p, loc['DATAHDR'])
    # Get the airmass
    loc['AIRMASS'] = spirouImage.GetAirmass(p, loc['DATAHDR'])
    # Get the Barycentric correction from header
    p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, loc['DATAHDR'])
    # set sources
    source = main_name + '+ spirouImage.ReadParams()'
    loc.set_sources(['OBJNAME', 'AIRMASS'], source)
    loc.set_sources(['OBJNAME', 'AIRMASS'], source)

    # ----------------------------------------------------------------------
    # Read wavelength solution
    # ----------------------------------------------------------------------
    # Force A and B to AB solution
    if p['FIBER'] in ['A', 'B']:
        wave_fiber = 'AB'
    else:
        wave_fiber = p['FIBER']
    # used for plotting only
    wout = spirouImage.GetWaveSolution(p,
                                       image=loc['DATA'],
                                       hdr=loc['DATAHDR'],
                                       return_wavemap=True,
                                       fiber=wave_fiber)
    _, loc['WAVE'], _ = wout
    loc.set_source('WAVE', main_name)

    # ----------------------------------------------------------------------
    # Get and Normalise the blaze
    # ----------------------------------------------------------------------
    p, loc = spirouTelluric.GetNormalizedBlaze(p, loc, loc['DATAHDR'])

    # ----------------------------------------------------------------------
    # Load transmission files
    # ----------------------------------------------------------------------
    transdata = spirouDB.GetDatabaseTellMap(p)
    trans_files = transdata[0]
    # make sure we have unique filenames for trans_files
    trans_files = np.unique(trans_files)

    # ----------------------------------------------------------------------
    # Start plotting
    # ----------------------------------------------------------------------
    if p['DRS_PLOT'] > 0:
        # start interactive plot
        sPlt.start_interactive_session(p)

    # ----------------------------------------------------------------------
    # Load template (if available)
    # ----------------------------------------------------------------------
    # read filename from telluDB
    template_file = spirouDB.GetDatabaseObjTemp(p,
                                                loc['OBJNAME'],
                                                required=False)
    # if we don't have a template flag it
    if template_file is None:
        loc['FLAG_TEMPLATE'] = False
        loc['TEMPLATE'] = None
    else:
        loc['FLAG_TEMPLATE'] = True
        # load template
        template, _, _, _ = spirouImage.ReadImage(p, template_file)
        # add to loc
        loc['TEMPLATE'] = template
    # set the source for flag and template
    loc.set_sources(['FLAG_TEMPLATE', 'TEMPLATE'], main_name)

    # ----------------------------------------------------------------------
    # load the expected atmospheric transmission
    # ----------------------------------------------------------------------
    # read filename from telluDB
    tapas_file_names = spirouDB.GetDatabaseTellConv(p)
    tapas_file_name = tapas_file_names[-1]
    # load atmospheric transmission
    loc['TAPAS_ALL_SPECIES'] = np.load(tapas_file_name)
    loc.set_source('TAPAS_ALL_SPECIES', main_name)

    # ----------------------------------------------------------------------
    # Generate the absorption map
    # ----------------------------------------------------------------------
    # get number of files
    nfiles = len(trans_files)
    npc = p['TELLU_NUMBER_OF_PRINCIPLE_COMP']
    # check that we have enough files (greater than number of principle
    #    components)
    if nfiles <= npc:
        emsg1 = 'Not enough "TELL_MAP" files in telluDB to run PCA analysis'
        emsg2 = '\tNumber of files = {0}, number of PCA components = {1}'
        emsg3 = '\tNumber of files > number of PCA components'
        emsg4 = '\tAdd more files or reduce number of PCA components'
        WLOG(p, 'error', [emsg1, emsg2.format(nfiles, npc), emsg3, emsg4])

    # check whether we can used pre-saved abso
    filetime = spirouImage.GetMostRecent(trans_files)
    tout = spirouConfig.Constants.TELLU_ABSO_SAVE(p, filetime)
    abso_save_file, absoprefix = tout
    use_saved = os.path.exists(abso_save_file)
    # noinspection PyBroadException
    try:
        # try loading from file
        abso = np.load(abso_save_file)
        # log progress
        wmsg = 'Loaded abso from file {0}'.format(abso_save_file)
        WLOG(p, '', wmsg)
    except:
        # set up storage for the absorption
        abso = np.zeros([nfiles, np.product(loc['DATA'].shape)])
        # loop around outputfiles and add them to abso
        for it, filename in enumerate(trans_files):
            # load data
            data_it, _, _, _ = spirouImage.ReadImage(p, filename=filename)
            # push data into array
            abso[it, :] = data_it.reshape(np.product(loc['DATA'].shape))
        # log progres
        wmsg = 'Saving abso to file {0}'.format(abso_save_file)
        WLOG(p, '', wmsg)
        # remove all abso save files (only need most recent one)
        afolder = os.path.dirname(abso_save_file)
        afilelist = os.listdir(afolder)
        for afile in afilelist:
            if afile.startswith(absoprefix):
                os.remove(os.path.join(afolder, afile))
        # save to file for later use
        np.save(abso_save_file, abso)

    # filter out rows of all NaNs
    # TODO: Why are we getting all NaN e2ds files?
    abso_filtered = []
    for row in range(len(abso)):
        if np.sum(np.isnan(abso[row])) != len(abso[row]):
            abso_filtered.append(abso[row])
        else:
            wargs = [trans_files[row]]
            WLOG(p, '', 'Removing trans file {0} (all NaN)'.format(*wargs))
    abso = np.array(abso_filtered)

    # log the absorption cube
    with warnings.catch_warnings(record=True) as w:
        log_abso = np.log(abso)

    # ----------------------------------------------------------------------
    # Locate valid pixels for PCA
    # ----------------------------------------------------------------------
    # determining the pixels relevant for PCA construction
    keep = np.isfinite(np.sum(abso, axis=0))
    # log fraction of valid (non NaN) pixels
    fraction = np.nansum(keep) / len(keep)
    wmsg = 'Fraction of valid pixels (not NaNs) for PCA construction = {0:.3f}'
    WLOG(p, '', wmsg.format(fraction))
    # log fraction of valid pixels > 1 - (1/e)
    with warnings.catch_warnings(record=True) as w:
        keep &= np.min(log_abso, axis=0) > -1
    fraction = np.nansum(keep) / len(keep)
    wmsg = 'Fraction of valid pixels with transmission > 1 - (1/e) = {0:.3f}'
    WLOG(p, '', wmsg.format(fraction))

    # ----------------------------------------------------------------------
    # Perform PCA analysis on the log of the telluric absorption map
    # ----------------------------------------------------------------------
    # Requires p:
    #           TELLU_NUMBER_OF_PRINCIPLE_COMP
    #           ADD_DERIV_PC
    #           FIT_DERIV_PC
    # Requires loc:
    #           DATA
    # Returns loc:
    #           PC
    #           NPC
    #           FIT_PC
    loc = spirouTelluric.CalculateAbsorptionPCA(p, loc, log_abso, keep)

    # Plot PCA components
    # debug plot
    if p['DRS_PLOT'] and (p['DRS_DEBUG'] > 1):
        # plot the transmission map plot
        sPlt.tellu_pca_comp_plot(p, loc)

    # ----------------------------------------------------------------------
    # Get master wavelength grid for shifting
    # ----------------------------------------------------------------------
    # get master wave map
    loc['MASTERWAVEFILE'] = spirouDB.GetDatabaseMasterWave(p)
    loc.set_source('MASTERWAVEFILE', main_name)
    # log progress
    wmsg1 = 'Getting master wavelength grid'
    wmsg2 = '\tFile = {0}'.format(os.path.basename(loc['MASTERWAVEFILE']))
    WLOG(p, '', [wmsg1, wmsg2])
    # Force A and B to AB solution
    if p['FIBER'] in ['A', 'B']:
        wave_fiber = 'AB'
    else:
        wave_fiber = p['FIBER']
    # read master wave map
    mout = spirouImage.GetWaveSolution(p,
                                       filename=loc['MASTERWAVEFILE'],
                                       return_wavemap=True,
                                       quiet=True,
                                       fiber=wave_fiber)
    _, loc['MASTERWAVE'], _ = mout
    loc.set_source('MASTERWAVE', main_name)

    # ----------------------------------------------------------------------
    # Loop around telluric files
    # ----------------------------------------------------------------------
    for basefilename in p['ARG_FILE_NAMES']:

        # ------------------------------------------------------------------
        # Construct absolute file path
        # ------------------------------------------------------------------
        filename = os.path.join(p['ARG_FILE_DIR'], basefilename)
        # ------------------------------------------------------------------
        # Construct output file names
        # ------------------------------------------------------------------
        outfile1, tag1 = CONSTANTS.TELLU_FIT_OUT_FILE(p, filename)
        outfilename1 = os.path.basename(outfile1)
        outfile2, tag2 = CONSTANTS.TELLU_FIT_RECON_FILE(p, filename)
        outfilename2 = os.path.basename(outfile2)

        # ------------------------------------------------------------------
        # Read filename
        # ------------------------------------------------------------------
        # read image
        tdata, thdr, _, _ = spirouImage.ReadImage(p, filename)
        # normalise with blaze function
        loc['SP'] = tdata / loc['NBLAZE']
        loc.set_source('SP', main_name)

        # ------------------------------------------------------------------
        # check that file has valid DPRTYPE
        # ------------------------------------------------------------------
        # get FP_FP DPRTYPE
        p = spirouImage.ReadParam(p, thdr, 'KW_DPRTYPE', 'DPRTYPE', dtype=str)
        # if dprtype is incorrect skip
        if p['DPRTYPE'] not in p['ALLOWED_TELLURIC_DPRTYPES']:
            wmsg1 = 'Skipping file (DPRTYPE incorrect)'
            wmsg2 = '\t DPRTYPE = {0}'.format(p['DPRTYPE'])
            WLOG(p, 'warning', [wmsg1, wmsg2])
            continue

        # ------------------------------------------------------------------
        # Set storage
        # ------------------------------------------------------------------
        loc['RECON_ABSO'] = np.ones(np.product(loc['DATA'].shape))
        loc['AMPS_ABSOL_TOTAL'] = np.zeros(loc['NPC'])
        loc.set_sources(['RECON_ABSO', 'AMPS_ABSOL_TOTAL'], main_name)

        # ------------------------------------------------------------------
        # Read wavelength solution
        # ------------------------------------------------------------------
        # Force A and B to AB solution
        if p['FIBER'] in ['A', 'B']:
            wave_fiber = 'AB'
        else:
            wave_fiber = p['FIBER']
        # get wavelength solution
        wout = spirouImage.GetWaveSolution(p,
                                           image=tdata,
                                           hdr=thdr,
                                           return_wavemap=True,
                                           return_filename=True,
                                           fiber=wave_fiber)
        _, loc['WAVE_IT'], loc['WAVEFILE'], loc['WSOURCE'] = wout
        loc.set_sources(['WAVE_IT', 'WAVEFILE', 'WSOURCE'], main_name)
        # load wave keys
        loc = spirouImage.GetWaveKeys(p, loc, thdr)

        # ------------------------------------------------------------------
        # Interpolate at shifted wavelengths (if we have a template)
        # ------------------------------------------------------------------
        if loc['FLAG_TEMPLATE']:
            # Requires p:
            #           TELLU_FIT_KEEP_FRAC
            # Requires loc:
            #           DATA
            #           TEMPLATE
            #           WAVE_IT
            # Returns:
            #           TEMPLATE2
            loc = spirouTelluric.BervCorrectTemplate(p, loc, thdr)

            # debug plot
            if p['DRS_PLOT'] and (p['DRS_DEBUG'] > 1):
                # plot the transmission map plot
                sPlt.tellu_fit_tellu_spline_plot(p, loc)

        # store PC and TAPAS_ALL_SPECIES before shift
        loc['PC_PRESHIFT'] = np.array(loc['PC'])
        loc['TAPAS_ALL_PRESHIFT'] = np.array(loc['TAPAS_ALL_SPECIES'])
        loc.set_sources(['PC_PRESHIFT', 'TAPAS_ALL_PRESHIFT'], main_name)

        # ------------------------------------------------------------------
        # Shift the pca components to correct frame
        # ------------------------------------------------------------------
        # log process
        wmsg1 = 'Shifting PCA components from master wavelength grid'
        wmsg2 = '\tFile = {0}'.format(os.path.basename(loc['MASTERWAVEFILE']))
        WLOG(p, '', [wmsg1, wmsg2])
        # shift pca components (one by one)
        for comp in range(loc['NPC']):
            wargs = [p, loc['PC'][:, comp], loc['MASTERWAVE'], loc['WAVE_IT']]
            shift_pc = spirouTelluric.Wave2Wave(*wargs, reshape=True)
            loc['PC'][:, comp] = shift_pc.reshape(wargs[1].shape)

            wargs = [
                p, loc['FIT_PC'][:, comp], loc['MASTERWAVE'], loc['WAVE_IT']
            ]
            shift_fpc = spirouTelluric.Wave2Wave(*wargs, reshape=True)
            loc['FIT_PC'][:, comp] = shift_fpc.reshape(wargs[1].shape)

        # ------------------------------------------------------------------
        # Shift the tapas spectrum to correct frame
        # ------------------------------------------------------------------
        # log process
        wmsg1 = 'Shifting TAPAS spectrum from master wavelength grid'
        wmsg2 = '\tFile = {0}'.format(os.path.basename(loc['MASTERWAVEFILE']))
        WLOG(p, '', [wmsg1, wmsg2])
        # shift tapas
        for comp in range(len(loc['TAPAS_ALL_SPECIES'])):
            wargs = [
                p, loc['TAPAS_ALL_SPECIES'][comp], loc['MASTERWAVE'],
                loc['WAVE_IT']
            ]
            stapas = spirouTelluric.Wave2Wave(*wargs, reshape=True)
            loc['TAPAS_ALL_SPECIES'][comp] = stapas.reshape(wargs[1].shape)

        # Debug plot to test shifting
        if p['DRS_PLOT'] and p['DRS_DEBUG'] > 1:
            sPlt.tellu_fit_debug_shift_plot(p, loc)

        # ------------------------------------------------------------------
        # Calculate reconstructed absorption
        # ------------------------------------------------------------------
        # Requires p:
        #           TELLU_FIT_MIN_TRANSMISSION
        #           TELLU_FIT_NITER
        #           TELLU_LAMBDA_MIN
        #           TELLU_LAMBDA_MAX
        #           TELLU_FIT_VSINI
        #           TRANSMISSION_CUT
        #           FIT_DERIV_PC
        #           LOG_OPT
        # Requires loc:
        #           FLAG_TEMPLATE
        #           TAPAS_ALL_SPECIES
        #           AMPS_ABSOL_TOTAL
        #           WAVE_IT
        #           TEMPLATE2
        #           FIT_PC
        #           NPC
        #           PC
        # Returns loc:
        #           SP2
        #           TEMPLATE2
        #           RECON_ABSO
        #           AMPS_ABSOL_TOTAL
        loc = spirouTelluric.CalcReconAbso(p, loc)
        # debug plot
        if p['DRS_PLOT'] > 0:
            # plot the recon abso plot
            sPlt.tellu_fit_recon_abso_plot(p, loc)

        # ------------------------------------------------------------------
        # Get molecular absorption
        # ------------------------------------------------------------------
        # Requires p:
        #           TELLU_FIT_LOG_LIMIT
        # Requeres loc:
        #           RECON_ABSO
        #           TAPAS_ALL_SPECIES
        # Returns loc:
        #           TAPAS_{molecule}
        loc = spirouTelluric.CalcMolecularAbsorption(p, loc)

        # ----------------------------------------------------------------------
        # Quality control
        # ----------------------------------------------------------------------
        # set passed variable and fail message list
        passed, fail_msg = True, []
        qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
        # ----------------------------------------------------------------------
        # get SNR for each order from header
        nbo = loc['DATA'].shape[0]
        snr_order = p['QC_FIT_TELLU_SNR_ORDER']
        snr = spirouImage.Read1Dkey(p, thdr, p['kw_E2DS_SNR'][0], nbo)
        # check that SNR is high enough
        if snr[snr_order] < p['QC_FIT_TELLU_SNR_MIN']:
            fmsg = 'low SNR in order {0}: ({1:.2f} < {2:.2f})'
            fargs = [snr_order, snr[snr_order], p['QC_FIT_TELLU_SNR_MIN']]
            fail_msg.append(fmsg.format(*fargs))
            passed = False
            qc_pass.append(0)
        else:
            qc_pass.append(1)
        # add to qc header lists
        qc_values.append(snr[snr_order])
        qc_name_str = 'SNR[{0}]'.format(snr_order)
        qc_names.append(qc_name_str)
        qc_logic.append('{0} < {1:.2f}'.format(qc_name_str,
                                               p['QC_FIT_TELLU_SNR_ORDER']))
        # ----------------------------------------------------------------------
        # finally log the failed messages and set QC = 1 if we pass the
        # quality control QC = 0 if we fail quality control
        if passed:
            WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
            p['QC'] = 1
            p.set_source('QC', __NAME__ + '/main()')
        else:
            for farg in fail_msg:
                wmsg = 'QUALITY CONTROL FAILED: {0}'
                WLOG(p, 'warning', wmsg.format(farg))
            p['QC'] = 0
            p.set_source('QC', __NAME__ + '/main()')
            continue
        # store in qc_params
        qc_params = [qc_names, qc_values, qc_logic, qc_pass]

        # ------------------------------------------------------------------
        # Get components amplitudes for header
        # ------------------------------------------------------------------
        # get raw file name
        raw_in_file = os.path.basename(p['FITSFILENAME'])
        # copy original keys
        hdict = spirouImage.CopyOriginalKeys(thdr)
        # add version number
        hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_DRS_DATE'],
                                   value=p['DRS_DATE'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_DATE_NOW'],
                                   value=p['DATE_NOW'])
        hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
        # set the input files
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_CDBBLAZE'],
                                   value=p['BLAZFILE'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_CDBWAVE'],
                                   value=loc['WAVEFILE'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_WAVESOURCE'],
                                   value=loc['WSOURCE'])
        hdict = spirouImage.AddKey1DList(p,
                                         hdict,
                                         p['KW_INFILE1'],
                                         dim1name='file',
                                         values=p['ARG_FILE_NAMES'])
        # add qc parameters
        hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
        hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
        # set tellu keys
        npc = loc['NPC']
        hdict = spirouImage.AddKey(p, hdict, p['KW_TELLU_NPC'], value=npc)
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_TELLU_FIT_DPC'],
                                   value=p['FIT_DERIV_PC'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_TELLU_ADD_DPC'],
                                   value=p['ADD_DERIV_PC'])
        if p['ADD_DERIV_PC']:
            values = loc['AMPS_ABSOL_TOTAL'][:npc - 2]
            hdict = spirouImage.AddKey1DList(p,
                                             hdict,
                                             p['KW_TELLU_AMP_PC'],
                                             values=values,
                                             dim1name='amp')
            hdict = spirouImage.AddKey(p,
                                       hdict,
                                       p['KW_TELLU_DV_TELL1'],
                                       value=loc['AMPS_ABSOL_TOTAL'][npc - 2])
            hdict = spirouImage.AddKey(p,
                                       hdict,
                                       p['KW_TELLU_DV_TELL2'],
                                       value=loc['AMPS_ABSOL_TOTAL'][npc - 1])
        else:
            values = loc['AMPS_ABSOL_TOTAL'][:npc]
            hdict = spirouImage.AddKey1DList(p,
                                             hdict,
                                             p['KW_TELLU_AMP_PC'],
                                             values=values,
                                             dim1name='PC')

        # ------------------------------------------------------------------
        # Write corrected spectrum to E2DS
        # ------------------------------------------------------------------
        # reform the E2DS
        sp_out = loc['SP2'] / loc['RECON_ABSO']
        sp_out = sp_out.reshape(loc['DATA'].shape)
        # multiply by blaze
        sp_out = sp_out * loc['NBLAZE']
        hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
        # log progress
        wmsg = 'Saving {0} to file'.format(outfilename1)
        WLOG(p, '', wmsg)
        # write sp_out to file
        p = spirouImage.WriteImage(p, outfile1, sp_out, hdict)

        # ------------------------------------------------------------------
        # 1-dimension spectral S1D (uniform in wavelength)
        # ------------------------------------------------------------------
        # get arguments for E2DS to S1D
        e2dsargs = [loc['WAVE'], sp_out, loc['BLAZE']]
        # get 1D spectrum
        xs1d1, ys1d1 = spirouImage.E2DStoS1D(p, *e2dsargs, wgrid='wave')
        # Plot the 1D spectrum
        if p['DRS_PLOT'] > 0:
            sPlt.ext_1d_spectrum_plot(p, xs1d1, ys1d1)
        # construct file name
        targs = [p, raw_in_file]
        s1dfile1, tag3 = spirouConfig.Constants.TELLU_FIT_S1D_FILE1(*targs)
        s1dfilename1 = os.path.basename(s1dfile1)
        # add header keys
        # set the version
        hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_DRS_DATE'],
                                   value=p['DRS_DATE'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_DATE_NOW'],
                                   value=p['DATE_NOW'])
        hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3)
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_EXT_TYPE'],
                                   value=p['DPRTYPE'])
        # log writing to file
        wmsg = 'Saving 1D spectrum (uniform in wavelength) in {0}'
        WLOG(p, '', wmsg.format(s1dfilename1))
        # Write to file
        columns = ['wavelength', 'flux', 'eflux']
        values = [xs1d1, ys1d1, np.zeros_like(ys1d1)]
        units = ['nm', None, None]
        s1d1 = spirouImage.MakeTable(p, columns, values, units=units)
        spirouImage.WriteTable(p, s1d1, s1dfile1, header=hdict)

        # ------------------------------------------------------------------
        # 1-dimension spectral S1D (uniform in velocity)
        # ------------------------------------------------------------------
        # get arguments for E2DS to S1D
        e2dsargs = [loc['WAVE'], sp_out, loc['BLAZE']]
        # get 1D spectrum
        xs1d2, ys1d2 = spirouImage.E2DStoS1D(p, *e2dsargs, wgrid='velocity')
        # Plot the 1D spectrum
        if p['DRS_PLOT'] > 0:
            sPlt.ext_1d_spectrum_plot(p, xs1d2, ys1d2)
        # construct file name
        targs = [p, raw_in_file]
        s1dfile2, tag4 = spirouConfig.Constants.TELLU_FIT_S1D_FILE2(*targs)
        s1dfilename2 = os.path.basename(s1dfile2)
        # add header keys
        hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_DRS_DATE'],
                                   value=p['DRS_DATE'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_DATE_NOW'],
                                   value=p['DATE_NOW'])
        hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4)
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_EXT_TYPE'],
                                   value=p['DPRTYPE'])
        # log writing to file
        wmsg = 'Saving 1D spectrum (uniform in velocity) in {0}'
        WLOG(p, '', wmsg.format(s1dfilename2))
        # Write to file
        columns = ['wavelength', 'flux', 'eflux']
        values = [xs1d2, ys1d2, np.zeros_like(ys1d2)]
        units = ['nm', None, None]
        s1d2 = spirouImage.MakeTable(p, columns, values, units=units)
        spirouImage.WriteTable(p, s1d2, s1dfile2, header=hdict)

        # ------------------------------------------------------------------
        # Write reconstructed absorption to E2DS
        # ------------------------------------------------------------------
        # set up empty storage
        recon_abso2 = np.zeros_like(loc['DATA'])
        # get dimensions of data
        ydim, xdim = loc['DATA'].shape
        # loop around orders
        for order_num in range(ydim):
            # get start and end points
            start, end = xdim * order_num, xdim * order_num + xdim
            # save to storage
            recon_abso2[order_num, :] = loc['RECON_ABSO'][start:end]
        # add molecular absorption to file
        for it, molecule in enumerate(p['TELLU_ABSORBERS'][1:]):
            # get molecule keyword store and key
            molkey = '{0}_{1}'.format(p['KW_TELLU_ABSO'][0], molecule.upper())
            molkws = [molkey, 0, 'Absorption in {0}'.format(molecule.upper())]
            # load into hdict
            hdict = spirouImage.AddKey(p, hdict, molkws, value=loc[molkey])
            # add water col
            if molecule == 'h2o':
                loc['WATERCOL'] = loc[molkey]
                # set source
                loc.set_source('WATERCOL', main_name)
        # add the tau keys
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_TAU_H2O'],
                                   value=loc['TAU_H2O'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_TAU_REST'],
                                   value=loc['TAU_REST'])
        # log progress
        wmsg = 'Saving {0} to file'.format(outfilename2)
        WLOG(p, '', wmsg)
        # write recon_abso to file
        hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2)
        p = spirouImage.WriteImage(p, outfile2, recon_abso2, hdict)

        # ------------------------------------------------------------------
        # Update the Telluric database
        # ------------------------------------------------------------------
        if p['QC']:
            # add TELLU_OBJ to telluric database
            oparams = dict(objname=loc['OBJNAME'],
                           berv=loc['BERV'],
                           airmass=loc['AIRMASS'],
                           watercol=loc['WATERCOL'])
            spirouDB.UpdateDatabaseTellObj(p, outfilename1, **oparams)
            # copy file to database
            spirouDB.PutTelluFile(p, outfile1)
            # add TELLU_RECON to telluric database
            # add TELLU_OBJ to telluric database
            oparams = dict(objname=loc['OBJNAME'],
                           berv=loc['BERV'],
                           airmass=loc['AIRMASS'],
                           watercol=loc['WATERCOL'])
            spirouDB.UpdateDatabaseTellRecon(p, outfilename2, **oparams)
            # copy file to database
            spirouDB.PutTelluFile(p, outfile2)

    # ----------------------------------------------------------------------
    # End plotting
    # ----------------------------------------------------------------------
    # debug plot
    if p['DRS_PLOT'] > 0:
        # end interactive session
        sPlt.end_interactive_session(p)

    # ----------------------------------------------------------------------
    # End Message
    # ----------------------------------------------------------------------
    p = spirouStartup.End(p)
    # return a copy of locally defined variables in the memory
    return dict(locals())
Example #33
0
>>> r = sdf.SDFReader()
>>> m = r.read('model://pr2/model.sdf')
>>> w = VRMLWriter()
>>> w.write(m, '/tmp/pr2.wrl')
"""

from . import model
from . import utils
import os
import sys
import time
import subprocess
import atexit
import logging
import warnings
with warnings.catch_warnings():
    warnings.simplefilter('ignore')
    from .thirdparty import transformations as tf
import math
import numpy
import copy
import jinja2
import uuid
try:
    import CORBA
    import CosNaming
    import OpenHRP
except ImportError:
    print "Unable to find CORBA and OpenHRP library."
    print "You can install the library by:"
    print "$ sudo add-apt-repository ppa:hrg/daily"
Example #34
0
        return out


try:
    np.array(5).astype(float, copy=False)
except TypeError:
    # Compat where astype accepted no copy argument
    def astype(array, dtype, copy=True):
        if not copy and array.dtype == dtype:
            return array
        return array.astype(dtype)
else:
    astype = np.ndarray.astype

try:
    with warnings.catch_warnings(record=True):
        # Don't raise the numpy deprecation warnings that appear in
        # 1.9, but avoid Python bug due to simplefilter('ignore')
        warnings.simplefilter('always')
        sp.csr_matrix([1.0, 2.0, 3.0]).max(axis=0)
except (TypeError, AttributeError):
    # in scipy < 14.0, sparse matrix min/max doesn't accept an `axis` argument
    # the following code is taken from the scipy 0.14 codebase

    def _minor_reduce(X, ufunc):
        major_index = np.flatnonzero(np.diff(X.indptr))
        if X.data.size == 0 and major_index.size == 0:
            # Numpy < 1.8.0 don't handle empty arrays in reduceat
            value = np.zeros_like(X.data)
        else:
            value = ufunc.reduceat(X.data, X.indptr[major_index])
Example #35
0
def main(argv):
    if not check_output('which ceph')[0]:
        logger.error("No 'ceph' command available. Run this script from node, which has ceph access")
        return

    # TODO: Logs from down OSD
    opts = parse_args(argv)
    res_q = Queue.Queue()
    run_q = Queue.Queue()

    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        out_folder = os.tempnam()

    os.makedirs(out_folder)

    setup_loggers(getattr(logging, opts.log_level),
                  os.path.join(out_folder, "log.txt"))

    global logger_ready
    logger_ready = True

    global SSH_OPTS
    SSH_OPTS = SSH_OPTS.format(opts.ssh_conn_timeout, opts.ssh_private_key)

    global USERNAME
    USERNAME = opts.username

    collector_settings = CollectSettings()
    map(collector_settings.disable, opts.disable)

    allowed_collectors = opts.collectors.split(',')
    collectors = []

    if CephDataCollector.name in allowed_collectors:
        ceph_collector = CephDataCollector(opts, collector_settings, res_q)
        collectors.append(ceph_collector)
    else:
        ceph_collector = None

    if NodeCollector.name in allowed_collectors:
        node_collector = NodeCollector(opts, collector_settings, res_q)
        collectors.append(node_collector)
    else:
        node_collector = None

    if NodeResourseUsageCollector.name in allowed_collectors:
        node_resource_collector = NodeResourseUsageCollector(opts, collector_settings, res_q)
    else:
        node_resource_collector = None

    if CephPerformanceCollector.name in allowed_collectors:
        if CephDataCollector.name not in allowed_collectors:
            logger.error("Can't collect performance info without ceph info collected")
            exit(1)
        else:
            ceph_performance_collector = CephPerformanceCollector(opts, collector_settings, res_q)
    else:
        ceph_performance_collector = None

    nodes = discover_nodes(opts)
    nodes['master'][None] = [{}]

    for role, nodes_with_args in nodes.items():
        if role == 'node':
            continue
        logger.info("Found %s hosts with role %s", len(nodes_with_args), role)
        logger.info("Found %s services with role %s",
                    sum(map(len, nodes_with_args.values())), role)

    logger.info("Found %s hosts total", len(nodes['node']))

    good_hosts = set(get_sshable_hosts(nodes['node'].keys()))
    bad_hosts = set(nodes['node'].keys()) - good_hosts

    if len(bad_hosts) != 0:
        logger.warning("Next hosts aren't awailable over ssh and would be skipped: %s",
                       ",".join(bad_hosts))

    res_q.put((True, "bad_hosts", 'json', json.dumps(list(bad_hosts))))

    new_nodes = collections.defaultdict(lambda: {})

    for role, role_objs in nodes.items():
        if role == 'master':
            new_nodes[role] = role_objs
        else:
            for node, args in role_objs.items():
                if node in good_hosts:
                    new_nodes[role][node] = args

    nodes = new_nodes

    # collect data at the beginning
    if node_resource_collector is not None:
        for node, _ in nodes['node'].items():
            run_q.put((node_resource_collector.collect_node, "", node, {}))

    for role, nodes_with_args in nodes.items():
        for collector in collectors:
            if hasattr(collector, 'collect_' + role):
                coll_func = getattr(collector, 'collect_' + role)
                for node, kwargs_list in nodes_with_args.items():
                    for kwargs in kwargs_list:
                        run_q.put((coll_func, "", node, kwargs))

    save_results_thread = threading.Thread(target=save_results_th_func,
                                           args=(opts, res_q, out_folder))
    save_results_thread.daemon = True
    save_results_thread.start()

    t1 = time.time()
    try:
        run_all(opts, run_q)

        # collect data at the end
        if node_resource_collector is not None:
            dt = opts.usage_collect_interval - (time.time() - t1)
            if dt > 0:
                logger.info("Will wait for {0} seconds for usage data collection".format(int(dt)))
                for i in range(int(dt / 0.1)):
                    time.sleep(0.1)
            logger.info("Start final usage collection")
            for node, _ in nodes['node'].items():
                run_q.put((node_resource_collector.collect_node, "", node, {}))
            run_all(opts, run_q)

        if ceph_performance_collector is not None:
            logger.info("Start performace monitoring.")
            with ceph_collector.osd_devs_lock:
                osd_devs = ceph_collector.osd_devs.copy()

            per_node = collections.defaultdict(lambda: [])
            for node, data_dev, j_dev in osd_devs.values():
                per_node[node].extend((data_dev, j_dev))

            # start monitoring
            for node, data in per_node.items():
                run_q.put((ceph_performance_collector.start_performance_monitoring,
                          "", node, {'osd_devs': data}))
            run_all(opts, run_q)

            dt = opts.performance_collect_seconds
            logger.info("Will wait for {0} seconds for performance data collection".format(int(dt)))
            for i in range(int(dt / 0.1)):
                time.sleep(0.1)

            # collect results
            for node, data in per_node.items():
                run_q.put((ceph_performance_collector.collect_performance_data,
                          "", node, {}))
            run_all(opts, run_q)
    except Exception:
        logger.exception("When collecting data:")
    finally:
        res_q.put(None)
        # wait till all data collected
        save_results_thread.join()

    if opts.result is None:
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            out_file = os.tempnam() + ".tar.gz"
    else:
        out_file = opts.result

    check_output("cd {0} ; tar -zcvf {1} *".format(out_folder, out_file))
    logger.info("Result saved into %r", out_file)
    if opts.log_level in ('WARNING', 'ERROR', "CRITICAL"):
        print "Result saved into %r" % (out_file,)

    if not opts.dont_remove_unpacked:
        shutil.rmtree(out_folder)
    else:
        logger.info("Temporary folder %r", out_folder)
        if opts.log_level in ('WARNING', 'ERROR', "CRITICAL"):
            print "Temporary folder %r" % (out_folder,)
Example #36
0
def rsa_encrypt(key, plaintext):
    pubkey = RSA.importKey(key)
    cipher = PKCS1_v1_5.new(pubkey)
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        return base64.b64encode(cipher.encrypt(plaintext))
Example #37
0
 def inner(*args, **kwargs):
     with warnings.catch_warnings(record=True):
         warnings.simplefilter("ignore")
         response = f(*args, **kwargs)
     return response
Example #38
0
    def __init__(self, filename):
        """Create an Overview object using an "*overview.dat" filename

        Parameters
        ----------
        filename : str
            - should be a valid "overview.dat" style filename (see Output/ascii.cxx)

        Notes
        -----
        We can currently parse the following attributes:
            Metallicity : float
                [ mass fraction ]
            Background Density : float
                [ g cm^-3 ]
            Background Temperature : float
                [ K ]
            With Cooling : bool
            Cooling Type : str
            Number of SNe : int
            Cluster Mass : float
                [g]
            seed : int
            Mass Loss : str

        """
        super(Overview, self).__init__()

        if "_" in filename:
            self.id = os.path.basename(filename).split("_")[0]
        else:
            self.id = ""
        self.dirname = os.path.dirname(filename)
        # Add trailing slash (if dirname isn't empty)
        self.dirname = os.path.join(self.dirname, "")

        inputs_filename = os.path.join(
            os.path.dirname(filename),
            os.path.basename(filename).replace("overview", "inputs"))
        self.inputs = Inputs(inputs_filename)

        # default, since earlier runs won't have this saved
        self.cluster_mass = 0
        self.cooling_type = "equilibrium"
        self.mass_loss = "none"

        f = open(filename, "r")
        for line in f:
            if "Metallicity" in line:
                self.metallicity = float(line.split()[1])
            elif "Background Density" in line:
                self.background_density = float(line.split()[2])
            elif "Background Temperature" in line:
                self.background_temperature = float(line.split()[2])
            elif "With cooling" in line:
                self.with_cooling = string_to_bool(line.split()[2])
            elif "Cooling Type" in line:
                self.cooling_type = line.split()[-1]
            elif "Number of SNe" in line:
                self.num_SNe = int(line.split()[-1])
            elif "Cluster Mass" in line:
                self.cluster_mass = float(line.split()[-1]) * M_solar
            elif "seed" in line:
                self.seed = int(line.split()[-1])
            elif "mass loss" in line:
                self.mass_loss = line.split()[-1]
        f.close()

        SNe_filename = os.path.join(
            os.path.dirname(filename),
            os.path.basename(filename).replace("overview", "SNe"))
        with warnings.catch_warnings():
            warnings.filterwarnings("ignore",
                                    message='loadtxt: Empty input file: "*"')
            SNe = np.loadtxt(SNe_filename, ndmin=2, usecols=range(5))
        if (SNe.shape[0] != self.num_SNe):
            raise ValueError("Number of SNe in datafile " +
                             "doesn't match number listed in overview file" +
                             " for file: " + filename)
        if self.num_SNe > 0:
            self.SNe_times = SNe[:, 0]
            self.SNe_initial_mass = SNe[:, 1]
            self.SNe_ejecta_mass = SNe[:, 2]
            self.SNe_ejecta_mass_Z = SNe[:, 3]
            self.SNe_wind_mass = SNe[:, 4]

            sorted_indices = np.argsort(self.SNe_times)
            self.SNe_times = self.SNe_times[sorted_indices]
            self.SNe_initial_mass = self.SNe_initial_mass[sorted_indices]
            self.SNe_ejecta_mass = self.SNe_ejecta_mass[sorted_indices]
            self.SNe_ejecta_mass_Z = self.SNe_ejecta_mass_Z[sorted_indices]
            self.SNe_wind_mass = self.SNe_wind_mass[sorted_indices]
        else:
            self.SNe_times = np.array([])
            self.SNe_initial_mass = np.array([])
            self.SNe_ejecta_mass = np.array([])
            self.SNe_ejecta_mass_Z = np.array([])
            self.SNe_wind_mass = np.array([])

        return
Example #39
0
def test_dimension_normalization():
    with warnings.catch_warnings(record=True) as w:
        assert Ray((1, 1), (2, 1, 2)) == Ray((1, 1, 0), (2, 1, 2))
        assert len(w) == 1
Example #40
0
    def test_iloc_getitem_frame(self):
        df = DataFrame(np.random.randn(10, 4), index=lrange(0, 20, 2),
                       columns=lrange(0, 8, 2))

        result = df.iloc[2]
        with catch_warnings(record=True):
            filterwarnings("ignore", "\\n.ix", DeprecationWarning)
            exp = df.ix[4]
        tm.assert_series_equal(result, exp)

        result = df.iloc[2, 2]
        with catch_warnings(record=True):
            filterwarnings("ignore", "\\n.ix", DeprecationWarning)
            exp = df.ix[4, 4]
        assert result == exp

        # slice
        result = df.iloc[4:8]
        with catch_warnings(record=True):
            filterwarnings("ignore", "\\n.ix", DeprecationWarning)
            expected = df.ix[8:14]
        tm.assert_frame_equal(result, expected)

        result = df.iloc[:, 2:3]
        with catch_warnings(record=True):
            filterwarnings("ignore", "\\n.ix", DeprecationWarning)
            expected = df.ix[:, 4:5]
        tm.assert_frame_equal(result, expected)

        # list of integers
        result = df.iloc[[0, 1, 3]]
        with catch_warnings(record=True):
            filterwarnings("ignore", "\\n.ix", DeprecationWarning)
            expected = df.ix[[0, 2, 6]]
        tm.assert_frame_equal(result, expected)

        result = df.iloc[[0, 1, 3], [0, 1]]
        with catch_warnings(record=True):
            filterwarnings("ignore", "\\n.ix", DeprecationWarning)
            expected = df.ix[[0, 2, 6], [0, 2]]
        tm.assert_frame_equal(result, expected)

        # neg indices
        result = df.iloc[[-1, 1, 3], [-1, 1]]
        with catch_warnings(record=True):
            filterwarnings("ignore", "\\n.ix", DeprecationWarning)
            expected = df.ix[[18, 2, 6], [6, 2]]
        tm.assert_frame_equal(result, expected)

        # dups indices
        result = df.iloc[[-1, -1, 1, 3], [-1, 1]]
        with catch_warnings(record=True):
            filterwarnings("ignore", "\\n.ix", DeprecationWarning)
            expected = df.ix[[18, 18, 2, 6], [6, 2]]
        tm.assert_frame_equal(result, expected)

        # with index-like
        s = Series(index=lrange(1, 5))
        result = df.iloc[s.index]
        with catch_warnings(record=True):
            filterwarnings("ignore", "\\n.ix", DeprecationWarning)
            expected = df.ix[[2, 4, 6, 8]]
        tm.assert_frame_equal(result, expected)
Example #41
0
def keep_stable_genes(expression, threshold=0.9, percentile=True, rank=True,
                      return_stability=False):
    """
    Removes genes in `expression` with differential stability < `threshold`

    Calculates the similarity of gene expression across brain regions for every
    pair of donors in `expression`. Similarity is averaged across donor pairs
    and genes whose mean similarity falls below `threshold` are removed.

    Parameters
    ----------
    expression : list of (R, G) pandas.DataFrame
        Where each entry is the microarray expression of `R` regions across `G`
        genes for a given donor
    threshold : [0, 1] float, optional
        Minimum required average similarity (e.g, correlation) across donors
        for a gene to be retained. Default: 0.1
    percentile : bool, optional
        Whether to treat `threshold` as a percentile instead of an absolute
        cutoff. For example, `threshold=0.9` and `percentile=True` would
        retain only those genes with a differential stability in the top 10% of
        all genes, whereas `percentile=False` would retain only those genes
        with differential stability > 0.9. Default: True
    rank : bool, optional
        Whether to calculate similarity as Spearman correlation instead of
        Pearson correlation. Default: True
    return_stability : bool, optional
        Whether to return stability estimates for each gene in addition to
        expression data. Default: False

    Returns
    -------
    expression : list of (R, Gr) pandas.DataFrame
        Microarray expression for `R` regions across `Gr` genes, where `Gr` is
        the number of retained genes
    stability : (G,) numpy.ndarray
        Stability (average correlation) of each gene across pairs of donors.
        Only returned if ``return_stability=True``
    """

    # get number of donors and number of genes
    num_subj = len(expression)
    num_gene = expression[0].shape[-1]

    # rank data, if necessary
    for_corr = expression if not rank else [e.rank() for e in expression]

    # get correlation of gene expression across regions for all donor pairs
    gene_corrs = np.zeros((num_gene, sum(range(num_subj))))
    for n, (s1, s2) in enumerate(itertools.combinations(range(num_subj), 2)):
        regions = np.intersect1d(for_corr[s1].dropna(axis=0, how='all').index,
                                 for_corr[s2].dropna(axis=0, how='all').index)
        gene_corrs[:, n] = utils.efficient_corr(for_corr[s1].loc[regions],
                                                for_corr[s2].loc[regions])

    # average similarity across donors (ignore NaNs)
    with warnings.catch_warnings():
        warnings.filterwarnings('ignore', category=RuntimeWarning,
                                message='Mean of empty slice')
        gene_corrs = np.nan_to_num(np.nanmean(gene_corrs, axis=1))

    # calculate absolute threshold if percentile is desired
    if percentile:
        threshold = np.percentile(gene_corrs, threshold * 100)
    keep_genes = gene_corrs >= threshold
    expression = [e.iloc[:, keep_genes] for e in expression]

    if return_stability:
        return expression, gene_corrs

    return expression
Example #42
0
    def test_iloc_mask(self):

        # GH 3631, iloc with a mask (of a series) should raise
        df = DataFrame(lrange(5), list('ABCDE'), columns=['a'])
        mask = (df.a % 2 == 0)
        msg = ("iLocation based boolean indexing cannot use an indexable as"
               " a mask")
        with pytest.raises(ValueError, match=msg):
            df.iloc[mask]
        mask.index = lrange(len(mask))
        msg = ("iLocation based boolean indexing on an integer type is not"
               " available")
        with pytest.raises(NotImplementedError, match=msg):
            df.iloc[mask]

        # ndarray ok
        result = df.iloc[np.array([True] * len(mask), dtype=bool)]
        tm.assert_frame_equal(result, df)

        # the possibilities
        locs = np.arange(4)
        nums = 2 ** locs
        reps = lmap(bin, nums)
        df = DataFrame({'locs': locs, 'nums': nums}, reps)

        expected = {
            (None, ''): '0b1100',
            (None, '.loc'): '0b1100',
            (None, '.iloc'): '0b1100',
            ('index', ''): '0b11',
            ('index', '.loc'): '0b11',
            ('index', '.iloc'): ('iLocation based boolean indexing '
                                 'cannot use an indexable as a mask'),
            ('locs', ''): 'Unalignable boolean Series provided as indexer '
                          '(index of the boolean Series and of the indexed '
                          'object do not match',
            ('locs', '.loc'): 'Unalignable boolean Series provided as indexer '
                              '(index of the boolean Series and of the '
                              'indexed object do not match',
            ('locs', '.iloc'): ('iLocation based boolean indexing on an '
                                'integer type is not available'),
        }

        # UserWarnings from reindex of a boolean mask
        with catch_warnings(record=True):
            simplefilter("ignore", UserWarning)
            result = dict()
            for idx in [None, 'index', 'locs']:
                mask = (df.nums > 2).values
                if idx:
                    mask = Series(mask, list(reversed(getattr(df, idx))))
                for method in ['', '.loc', '.iloc']:
                    try:
                        if method:
                            accessor = getattr(df, method[1:])
                        else:
                            accessor = df
                        ans = str(bin(accessor[mask]['nums'].sum()))
                    except Exception as e:
                        ans = str(e)

                    key = tuple([idx, method])
                    r = expected.get(key)
                    if r != ans:
                        raise AssertionError(
                            "[%s] does not match [%s], received [%s]"
                            % (key, ans, r))
Example #43
0
    def on_episode_end(self, episode, logs):
        """ Compute and print training statistics of the episode when done """
        duration = timeit.default_timer() - self.episode_start[episode]
        episode_steps = len(self.observations[episode])

        # Format all metrics.
        metrics = np.array(self.metrics[episode])
        metrics_template = ''
        metrics_variables = []
        with warnings.catch_warnings():
            warnings.filterwarnings('error')
            for idx, name in enumerate(self.metrics_names):
                if idx > 0:
                    metrics_template += ', '
                try:
                    value = np.nanmean(metrics[:, idx])
                    metrics_template += '{}: {:f}'
                except Warning:
                    value = '--'
                    metrics_template += '{}: {}'
                metrics_variables += [name, value]          
        metrics_text = metrics_template.format(*metrics_variables)

        nb_step_digits = str(int(np.ceil(np.log10(self.params['nb_steps']))) + 1)

        lam = lambda l,n : np.array([d[n] for d in l])
        keys = self.observations[episode][0].keys()

        template = '{step: ' + nb_step_digits + 'd}/{nb_steps}: episode: {episode}, duration: {duration:.3f}s, episode steps: {episode_steps}, steps per second: {sps:.0f}, episode reward: {episode_reward:.3f}, mean reward: {reward_mean:.3f} [{reward_min:.3f}, {reward_max:.3f}], episode delay: {episode_delay:.3f}, mean delay: {delay_mean:.3f} [{delay_min:.3f}, {delay_max:.3f}], mean action: {action_mean:.3f} [{action_min:.3f}, {action_max:.3f}]' #'{step: ' + nb_step_digits + 'd}/{nb_steps}: episode: {episode}, duration: {duration:.3f}s, episode steps: {episode_steps}, steps per second: {sps:.0f}, episode reward: {episode_reward:.3f}, mean reward: {reward_mean:.3f} [{reward_min:.3f}, {reward_max:.3f}], mean action: {action_mean:.3f} [{action_min:.3f}, {action_max:.3f}], mean observation: {obs_mean:.3f} [{obs_min:.3f}, {obs_max:.3f}], {metrics}'
        for k in keys:
        	template += ', mean observation '+k+': {obs_mean_'+k+':.3f} [{obs_min_'+k+':.3f}, {obs_max_'+k+':.3f}]'
        template += ', {metrics}'

        variables = {
            'step': self.step,
            'nb_steps': self.params['nb_steps'],
            'episode': episode + 1,
            'duration': duration,
            'episode_steps': episode_steps,
            'sps': float(episode_steps) / duration,
            'episode_reward': np.sum(self.rewards[episode]),
            'reward_mean': np.mean(self.rewards[episode]),
            'reward_min': np.min(self.rewards[episode]),
            'reward_max': np.max(self.rewards[episode]),
            'episode_delay': np.sum(self.delays[episode]),
            'delay_mean': np.mean(self.delays[episode]),
            'delay_min': np.min(self.delays[episode]),
            'delay_max': np.max(self.delays[episode]),
            'action_mean': np.mean(self.actions[episode]),
            'action_min': np.min(self.actions[episode]),
            'action_max': np.max(self.actions[episode]),
            'metrics': metrics_text
        }
        for k in keys:
        	variables['obs_mean_'+k] = np.mean(lam(self.observations[episode], k))
        	variables['obs_min_'+k] = np.min(lam(self.observations[episode], k))
        	variables['obs_max_'+k] = np.max(lam(self.observations[episode], k))

        print(template.format(**variables))

        # Debug csv.
        with open("debug.csv", "a") as file:
          file.write(str("\n")+str(variables['episode_reward'])+str(",")+str(variables['reward_mean'])+str(",")+str(variables['episode_delay'])+str(",")+str(variables['delay_mean']))
          file.close()

        # Model checkpoint.
        if np.sum(self.rewards[episode])>self.lastreward:
          previousWeights = 'checkpoint_reward_{}.h5f'.format(self.lastreward)
          if os.path.exists(previousWeights): os.remove(previousWeights)
          self.lastreward = np.sum(self.rewards[episode])
          print("The reward is higher than the best one, saving checkpoint weights")
          newWeights = 'checkpoint_reward_{}.h5f'.format(np.sum(self.rewards[episode]))
          #self.model.save_weights(newWeights, overwrite=True)
          self.model.model.save(newWeights, overwrite=True)
        else:
          print("The reward is lower than the best one, checkpoint weights not updated")

        # Free up resources.
        del self.episode_start[episode]
        del self.observations[episode]
        del self.rewards[episode]
        del self.actions[episode]
        del self.metrics[episode]
        del self.delays[episode]
Example #44
0
def setUpModule():
    global WARNING_CONTEXT
    WARNING_CONTEXT = warnings.catch_warnings()
    WARNING_CONTEXT.__enter__()
    warnings.simplefilter('ignore', ResourceWarning)
Example #45
0
def etree_parse(path):
    # type: (str) -> Any
    with warnings.catch_warnings(record=False):
        warnings.filterwarnings("ignore", category=DeprecationWarning)
        return ElementTree.parse(path)
Example #46
0
    def report(self, tracker_names):
        assert isinstance(tracker_names, (list, tuple))

        # function for loading results
        def read_record(filename):
            with open(filename) as f:
                record = f.read().strip().split('\n')
            record = [[float(t) for t in line.split(',')]
                      for line in record]
            return record

        # assume tracker_names[0] is your tracker
        report_dir = os.path.join(self.report_dir, tracker_names[0])
        if not os.path.exists(report_dir):
            os.makedirs(report_dir)
        report_file = os.path.join(report_dir, 'performance.json')

        performance = {}
        for name in tracker_names:
            print('Evaluating', name)
            ious = {}
            ious_full = {}
            failures = {}
            times = {}
            masks = {}  # frame masks for attribute tags

            for s, (img_files, anno, meta) in enumerate(self.dataset):
                seq_name = self.dataset.seq_names[s]

                # initialize frames scores
                frame_num = len(img_files)
                ious[seq_name] = np.full(
                    (self.repetitions, frame_num), np.nan, dtype=float)
                ious_full[seq_name] = np.full(
                    (self.repetitions, frame_num), np.nan, dtype=float)
                failures[seq_name] = np.full(
                    (self.repetitions, frame_num), np.nan, dtype=float)
                times[seq_name] = np.full(
                    (self.repetitions, frame_num), np.nan, dtype=float)

                # read results of all repetitions
                record_files = sorted(glob.glob(os.path.join(
                    self.result_dir, name, 'baseline', seq_name,
                    '%s_[0-9]*.txt' % seq_name)))
                boxes = [read_record(f) for f in record_files]
                assert all([len(b) == len(anno) for b in boxes])

                # calculate frame ious with burnin
                bound = Image.open(img_files[0]).size
                seq_ious = [self._calc_iou(b, anno, bound, burnin=True)
                            for b in boxes]
                ious[seq_name][:len(seq_ious), :] = seq_ious

                # calculate frame ious without burnin
                seq_ious_full = [self._calc_iou(b, anno, bound)
                                 for b in boxes]
                ious_full[seq_name][:len(seq_ious_full), :] = seq_ious_full

                # calculate frame failures
                seq_failures = [
                    [len(b) == 1 and b[0] == 2 for b in boxes_per_rep]
                    for boxes_per_rep in boxes]
                failures[seq_name][:len(seq_failures), :] = seq_failures

                # collect frame runtimes
                time_file = os.path.join(
                    self.result_dir, name, 'baseline', seq_name,
                    '%s_time.txt' % seq_name)
                if os.path.exists(time_file):
                    seq_times = np.loadtxt(time_file, delimiter=',').T
                    times[seq_name][:len(seq_times), :] = seq_times

                # collect attribute masks
                tag_num = len(self.tags)
                masks[seq_name] = np.zeros((tag_num, frame_num), bool)
                for i, tag in enumerate(self.tags):
                    if tag in meta:
                        masks[seq_name][i, :] = meta[tag]
                # frames with no tags
                if 'empty' in self.tags:
                    tag_frames = np.array([
                        v for k, v in meta.items()
                        if not 'practical' in k], dtype=bool)
                    ind = self.tags.index('empty')
                    masks[seq_name][ind, :] = \
                        ~np.logical_or.reduce(tag_frames, axis=0)

            # concatenate frames
            seq_names = self.dataset.seq_names
            masks = np.concatenate(
                [masks[s] for s in seq_names], axis=1)
            ious = np.concatenate(
                [ious[s] for s in seq_names], axis=1)
            failures = np.concatenate(
                [failures[s] for s in seq_names], axis=1)

            with warnings.catch_warnings():
                # average over repetitions
                warnings.simplefilter('ignore', category=RuntimeWarning)
                ious = np.nanmean(ious, axis=0)
                failures = np.nanmean(failures, axis=0)
            
                # calculate average overlaps and failures for each tag
                tag_ious = np.array(
                    [np.nanmean(ious[m]) for m in masks])
                tag_failures = np.array(
                    [np.nansum(failures[m]) for m in masks])
                tag_frames = masks.sum(axis=1)

            # remove nan values
            tag_ious[np.isnan(tag_ious)] = 0.0
            tag_weights = tag_frames / tag_frames.sum()

            # calculate weighted accuracy and robustness
            accuracy = np.sum(tag_ious * tag_weights)
            robustness = np.sum(tag_failures * tag_weights)

            # calculate tracking speed
            times = np.concatenate([
                t.reshape(-1) for t in times.values()])
            # remove invalid values
            times = times[~np.isnan(times)]
            times = times[times > 0]
            if len(times) > 0:
                speed = np.mean(1. / times)
            else:
                speed = -1

            performance.update({name: {
                'accuracy': accuracy,
                'robustness': robustness,
                'speed_fps': speed}})

        # save performance
        with open(report_file, 'w') as f:
            json.dump(performance, f, indent=4)
        print('Performance saved at', report_file)

        return performance
Example #47
0
def test_main():
    with warnings.catch_warnings():
        warnings.filterwarnings("error", module="<test string>")
        run_unittest(GlobalTests)
Example #48
0
def platform(aliased=0, terse=0):

    """ Returns a single string identifying the underlying platform
        with as much useful information as possible (but no more :).

        The output is intended to be human readable rather than
        machine parseable. It may look different on different
        platforms and this is intended.

        If "aliased" is true, the function will use aliases for
        various platforms that report system names which differ from
        their common names, e.g. SunOS will be reported as
        Solaris. The system_alias() function is used to implement
        this.

        Setting terse to true causes the function to return only the
        absolute minimum information needed to identify the platform.

    """
    result = _platform_cache.get((aliased, terse), None)
    if result is not None:
        return result

    # Get uname information and then apply platform specific cosmetics
    # to it...
    system, node, release, version, machine, processor = uname()
    if machine == processor:
        processor = ''
    if aliased:
        system, release, version = system_alias(system, release, version)

    if system == 'Windows':
        # MS platforms
        rel, vers, csd, ptype = win32_ver(version)
        if terse:
            platform = _platform(system, release)
        else:
            platform = _platform(system, release, version, csd)

    elif system in ('Linux',):
        # Linux based systems
        with warnings.catch_warnings():
            # see issue #1322 for more information
            warnings.filterwarnings(
                'ignore',
                r'dist\(\) and linux_distribution\(\) '
                'functions are deprecated .*',
                PendingDeprecationWarning,
            )
            distname, distversion, distid = dist('')
        if distname and not terse:
            platform = _platform(system, release, machine, processor,
                                 'with',
                                 distname, distversion, distid)
        else:
            # If the distribution name is unknown check for libc vs. glibc
            libcname, libcversion = libc_ver(sys.executable)
            platform = _platform(system, release, machine, processor,
                                 'with',
                                 libcname+libcversion)
    elif system == 'Java':
        # Java platforms
        r, v, vminfo, (os_name, os_version, os_arch) = java_ver()
        if terse or not os_name:
            platform = _platform(system, release, version)
        else:
            platform = _platform(system, release, version,
                                 'on',
                                 os_name, os_version, os_arch)

    elif system == 'MacOS':
        # MacOS platforms
        if terse:
            platform = _platform(system, release)
        else:
            platform = _platform(system, release, machine)

    else:
        # Generic handler
        if terse:
            platform = _platform(system, release)
        else:
            bits, linkage = architecture(sys.executable)
            platform = _platform(system, release, machine,
                                 processor, bits, linkage)

    _platform_cache[(aliased, terse)] = platform
    return platform
def train_exact_gp(trainX, trainY, testX, testY, kind, model_kwargs, train_kwargs, devices=('cpu',),
                   skip_posterior_variances=False, skip_random_restart=False, evaluate_on_train=True,
                   output_device=None, record_pred_unc=False, double=False):
    """Create and train an exact GP with the given options"""
    model_kwargs = copy.copy(model_kwargs)
    train_kwargs = copy.copy(train_kwargs)
    d = trainX.shape[-1]
    devices = [torch.device(device) for device in devices]
    if output_device is None:
        output_device = devices[0]
    else:
        output_device = torch.device(output_device)
    type_ = torch.double if double else torch.float

    trainX = trainX.to(output_device, type_)
    trainY = trainY.to(output_device, type_)
    testX = testX.to(output_device, type_)
    testY = testY.to(output_device, type_)

    # replace with value from dataset for convenience
    for k, v in list(model_kwargs.items()):
        if isinstance(v, str) and v == 'd':
            model_kwargs[k] = d

    # Change some options just for initial training with random restarts.
    random_restarts = train_kwargs.pop('random_restarts', 1)
    init_iters = train_kwargs.pop('init_iters', 20)
    optimizer_ = _map_to_optim(train_kwargs.pop('optimizer'))
    rr_check_conv = train_kwargs.pop('rr_check_conv', False)

    initial_train_kwargs = copy.copy(train_kwargs)
    initial_train_kwargs['max_iter'] = init_iters
    initial_train_kwargs['check_conv'] = rr_check_conv
    # initial_train_kwargs['verbose'] = 0  # don't shout about it
    best_model, best_likelihood, best_mll = None, None, None
    best_loss = np.inf

    # TODO: move random restarts code to a train_to_convergence-like function
    if not skip_random_restart:
        # Do some number of random restarts, keeping the best one after a truncated training.
        for restart in range(random_restarts):
            # TODO: log somehow what's happening in the restarts.
            model, likelihood = create_exact_gp(trainX, trainY, kind, devices=devices, **model_kwargs)
            model = model.to(output_device, type_)

            # regular marginal log likelihood
            mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood, model)
            _ = train_to_convergence(model, trainX, trainY, optimizer=optimizer_,
                                     objective=mll, isloss=False, **initial_train_kwargs)
            model.train()
            output = model(trainX)
            loss = -mll(output, trainY).item()
            if loss < best_loss:
                best_loss = loss
                best_model = model
                best_likelihood = likelihood
                best_mll = mll
        model = best_model
        likelihood = best_likelihood
        mll = best_mll
    else:
        model, likelihood = create_exact_gp(trainX, trainY, kind, devices=devices, **model_kwargs)
        model = model.to(output_device, type_)
        mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood, model)

    # fit GP
    with warnings.catch_warnings(record=True) as w:
        trained_epochs = train_to_convergence(model, trainX, trainY, optimizer=optimizer_,
                                              objective=mll, isloss=False, **train_kwargs)

    model.eval()
    likelihood.eval()
    mll.eval()

    model_metrics = dict()
    model_metrics['trained_epochs'] = trained_epochs
    with torch.no_grad():
        model.train()  # consider prior for evaluation on train dataset
        likelihood.train()
        train_outputs = model(trainX)
        model_metrics['prior_train_nmll'] = -mll(train_outputs, trainY).item()

        with gpytorch.settings.skip_posterior_variances(skip_posterior_variances):
            model.eval()  # Now consider posterior distributions
            likelihood.eval()
            if evaluate_on_train:
                train_outputs = model(trainX)
                model_metrics['train_mse'] = mean_squared_error(train_outputs.mean, trainY)


            with warnings.catch_warnings(record=True) as w2:
                test_outputs = model(testX)
                pred_mean = test_outputs.mean


            if not skip_posterior_variances:
                # model_metrics['train_nll'] = -likelihood(train_outputs).log_prob(
                #     trainY).item()
                # model_metrics['test_nll'] = -likelihood(test_outputs).log_prob(
                #     testY).item()
                if evaluate_on_train:
                    model_metrics['train_nll'] = -mll(train_outputs, trainY).item()
                model_metrics['test_nll'] = -mll(test_outputs, testY).item()
                distro = likelihood(test_outputs)
                lower, upper = distro.confidence_region()
                frac = ((testY > lower) * (testY < upper)).to(torch.float).mean().item()
                model_metrics['test_pred_frac_in_cr'] = frac
                if record_pred_unc:
                    model_metrics['test_pred_z_score'] = (testY - distro.mean) / distro.stddev
                    # model_metrics['test_pred_var'] = distro.variance.tolist()
                    # model_metrics['test_pred_mean'] = distro.mean.tolist()

    model_metrics['training_warnings'] = len(w)
    model_metrics['testing_warning'] = '' if len(w2) == 0 else w2[-1].message
    model_metrics['state_dict_file'] = _save_state_dict(model)

    return model_metrics, pred_mean.to('cpu', torch.float), model
Example #50
0
def main():
    """
    Launches text to speech (inference).
    Inference is executed on a single GPU.
    """

    torch.backends.cudnn.benchmark = True

    parser = argparse.ArgumentParser(description='PyTorch FastPitch Inference',
                                     allow_abbrev=False)
    parser = parse_args(parser)
    args, unk_args = parser.parse_known_args()

    if args.output is not None:
        Path(args.output).mkdir(parents=False, exist_ok=True)

    log_fpath = args.log_file or str(Path(args.output, 'nvlog_infer.json'))
    log_fpath = unique_dllogger_fpath(log_fpath)
    DLLogger.init(backends=[JSONStreamBackend(Verbosity.DEFAULT, log_fpath),
                            StdOutBackend(Verbosity.VERBOSE)])
    [DLLogger.log("PARAMETER", {k:v}) for k,v in vars(args).items()]

    device = torch.device('cuda' if args.cuda else 'cpu')

    if args.fastpitch is not None:
        generator = load_and_setup_model(
            'FastPitch', parser, args.fastpitch, args.amp, device,
            unk_args=unk_args, forward_is_infer=True, ema=args.ema,
            jitable=args.torchscript)

        if args.torchscript:
            generator = torch.jit.script(generator)
    else:
        generator = None

    if args.waveglow is not None:
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            waveglow = load_and_setup_model(
                'WaveGlow', parser, args.waveglow, args.amp, device,
                unk_args=unk_args, forward_is_infer=True, ema=args.ema)
        denoiser = Denoiser(waveglow).to(device)
        waveglow = getattr(waveglow, 'infer', waveglow)
    else:
        waveglow = None

    if len(unk_args) > 0:
        raise ValueError(f'Invalid options {unk_args}')

    fields = load_fields(args.input)
    batches = prepare_input_sequence(
        fields, device, args.batch_size, args.dataset_path,
        load_mels=(generator is None))

    if args.include_warmup:
        # Use real data rather than synthetic - FastPitch predicts len
        for i in range(3):
            with torch.no_grad():
                if generator is not None:
                    b = batches[0]
                    mel, *_ = generator(b['text'], b['text_lens'])
                if waveglow is not None:
                    audios = waveglow(mel, sigma=args.sigma_infer).float()
                    _ = denoiser(audios, strength=args.denoising_strength)

    gen_measures = MeasureTime()
    waveglow_measures = MeasureTime()

    gen_kw = {'pace': args.pace,
              'pitch_tgt': None,
              'pitch_transform': build_pitch_transformation(args)}

    if args.torchscript:
        gen_kw.pop('pitch_transform')

    all_utterances = 0
    all_samples = 0
    all_letters = 0
    all_frames = 0

    reps = args.repeats
    log_enabled = True  # reps == 1
    log = lambda s, d: DLLogger.log(step=s, data=d) if log_enabled else None

    # for repeat in (tqdm.tqdm(range(reps)) if reps > 1 else range(reps)):
    k=0
    for rep in range(reps):
        for b in batches:
            if generator is None:
                log(rep, {'Synthesizing from ground truth mels'})
                mel, mel_lens = b['mel'], b['mel_lens']
            else:
                with torch.no_grad(), gen_measures:
                    mel, mel_lens, *_ = generator(
                        b['text'], b['text_lens'], **gen_kw)

                gen_infer_perf = mel.size(0) * mel.size(2) / gen_measures[-1]
                all_letters += b['text_lens'].sum().item()
                all_frames += mel.size(0) * mel.size(2)
                
                mel_fname = b['output'][k] if 'output' in b else f'audio_{k}.pt'
                mel_path = Path(args.output, mel_fname)               
                torch.save(mel.cpu(), mel_path)
                log(rep, {"fastpitch_frames_per_sec": gen_infer_perf})
                log(rep, {"fastpitch_latency": gen_measures[-1]})
                k=k+1

            if waveglow is not None:
                with torch.no_grad(), waveglow_measures:
                    audios = waveglow(mel, sigma=args.sigma_infer)
                    audios = denoiser(audios.float(),
                                      strength=args.denoising_strength
                                     ).squeeze(1)

                all_utterances += len(audios)
                all_samples += sum(audio.size(0) for audio in audios)
                waveglow_infer_perf = (
                    audios.size(0) * audios.size(1) / waveglow_measures[-1])

                log(rep, {"waveglow_samples_per_sec": waveglow_infer_perf})
                log(rep, {"waveglow_latency": waveglow_measures[-1]})

                if args.output is not None and reps == 1:
                    for i, audio in enumerate(audios):
                        audio = audio[:mel_lens[i].item() * args.stft_hop_length]

                        if args.fade_out:
                            fade_len = args.fade_out * args.stft_hop_length
                            fade_w = torch.linspace(1.0, 0.0, fade_len)
                            audio[-fade_len:] *= fade_w.to(audio.device)

                        audio = audio/torch.max(torch.abs(audio))
                        fname = b['output'][i] if 'output' in b else f'audio_{i}.wav'
                        audio_path = Path(args.output, fname)
                        write(audio_path, args.sampling_rate, audio.cpu().numpy())

            if generator is not None and waveglow is not None:
                log(rep, {"latency": (gen_measures[-1] + waveglow_measures[-1])})

    log_enabled = True
    if generator is not None:
        gm = np.sort(np.asarray(gen_measures))
        rtf = all_samples / (all_utterances * gm.mean() * args.sampling_rate)
        log('avg', {"fastpitch letters/s": all_letters / gm.sum()})
        log('avg', {"fastpitch_frames/s": all_frames / gm.sum()})
        log('avg', {"fastpitch_latency": gm.mean()})
        log('avg', {"fastpitch RTF": rtf})
        log('90%', {"fastpitch_latency": gm.mean() + norm.ppf((1.0 + 0.90) / 2) * gm.std()})
        log('95%', {"fastpitch_latency": gm.mean() + norm.ppf((1.0 + 0.95) / 2) * gm.std()})
        log('99%', {"fastpitch_latency": gm.mean() + norm.ppf((1.0 + 0.99) / 2) * gm.std()})
    if waveglow is not None:
        wm = np.sort(np.asarray(waveglow_measures))
        rtf = all_samples / (all_utterances * wm.mean() * args.sampling_rate)
        log('avg', {"waveglow_samples/s": all_samples / wm.sum()})
        log('avg', {"waveglow_latency": wm.mean()})
        log('avg', {"waveglow RTF": rtf})
        log('90%', {"waveglow_latency": wm.mean() + norm.ppf((1.0 + 0.90) / 2) * wm.std()})
        log('95%', {"waveglow_latency": wm.mean() + norm.ppf((1.0 + 0.95) / 2) * wm.std()})
        log('99%', {"waveglow_latency": wm.mean() + norm.ppf((1.0 + 0.99) / 2) * wm.std()})
    if generator is not None and waveglow is not None:
        m = gm + wm
        rtf = all_samples / (all_utterances * m.mean() * args.sampling_rate)
        log('avg', {"samples/s": all_samples / m.sum()})
        log('avg', {"letters/s": all_letters / m.sum()})
        log('avg', {"latency": m.mean()})
        log('avg', {"RTF": rtf})
        log('90%', {"latency": m.mean() + norm.ppf((1.0 + 0.90) / 2) * m.std()})
        log('95%', {"latency": m.mean() + norm.ppf((1.0 + 0.95) / 2) * m.std()})
        log('99%', {"latency": m.mean() + norm.ppf((1.0 + 0.99) / 2) * m.std()})
    DLLogger.flush()
def test_ec(ac=None, rd=None):
    """Test EC methods."""
    if ac is None:
        # test type/value checking for audio_controller
        assert_raises(TypeError,
                      ExperimentController,
                      *std_args,
                      audio_controller=1,
                      stim_fs=44100,
                      **std_kwargs)
        assert_raises(ValueError,
                      ExperimentController,
                      *std_args,
                      audio_controller='foo',
                      stim_fs=44100,
                      **std_kwargs)
        assert_raises(ValueError,
                      ExperimentController,
                      *std_args,
                      audio_controller=dict(TYPE='foo'),
                      stim_fs=44100,
                      **std_kwargs)
        # monitor, etc.
        assert_raises(TypeError,
                      ExperimentController,
                      *std_args,
                      monitor='foo',
                      **std_kwargs)
        assert_raises(KeyError,
                      ExperimentController,
                      *std_args,
                      monitor=dict(),
                      **std_kwargs)
        assert_raises(ValueError,
                      ExperimentController,
                      *std_args,
                      response_device='foo',
                      **std_kwargs)
        std_kwargs.update(window_size=10.)
        assert_raises(ValueError, ExperimentController, *std_args,
                      **std_kwargs)
        std_kwargs.update(window_size=(1, 1))
        assert_raises(ValueError,
                      ExperimentController,
                      *std_args,
                      audio_controller='pyglet',
                      response_device='tdt',
                      **std_kwargs)
        assert_raises(ValueError,
                      ExperimentController,
                      *std_args,
                      audio_controller='pyglet',
                      response_device='keyboard',
                      trigger_controller='tdt',
                      **std_kwargs)

        # test type checking for 'session'
        std_kwargs['session'] = 1
        assert_raises(TypeError,
                      ExperimentController,
                      *std_args,
                      audio_controller='pyglet',
                      stim_fs=44100,
                      **std_kwargs)
        std_kwargs['session'] = '01'

        # test value checking for trigger controller
        assert_raises(ValueError,
                      ExperimentController,
                      *std_args,
                      audio_controller='pyglet',
                      trigger_controller='foo',
                      stim_fs=44100,
                      **std_kwargs)

        # test value checking for RMS checker
        assert_raises(ValueError,
                      ExperimentController,
                      *std_args,
                      audio_controller='pyglet',
                      check_rms=True,
                      stim_fs=44100,
                      **std_kwargs)

        # run rest of test with audio_controller == 'pyglet'
        this_ac = 'pyglet'
        this_rd = 'keyboard'
        this_tc = 'dummy'
        this_fs = 44100
    else:
        assert ac == 'tdt'
        # run rest of test with audio_controller == 'tdt'
        this_ac = ac
        this_rd = rd
        this_tc = ac
        this_fs = get_tdt_rates()['25k']
        assert_raises(ValueError,
                      ExperimentController,
                      *std_args,
                      audio_controller=dict(TYPE=this_ac, TDT_MODEL='foo'),
                      **std_kwargs)
    with warnings.catch_warnings(record=True) as w:
        for suppress in (True, False):
            with ExperimentController(*std_args,
                                      audio_controller=this_ac,
                                      response_device=this_rd,
                                      trigger_controller=this_tc,
                                      stim_fs=100.,
                                      suppress_resamp=suppress,
                                      **std_kwargs) as ec:
                pass
    warnings.simplefilter('ignore')  # ignore dummy TDT warning
    with ExperimentController(*std_args,
                              audio_controller=this_ac,
                              response_device=this_rd,
                              trigger_controller=this_tc,
                              stim_fs=this_fs,
                              **std_kwargs) as ec:
        warnings.simplefilter('always')
        assert_true(ec.participant == std_kwargs['participant'])
        assert_true(ec.session == std_kwargs['session'])
        assert_true(ec.exp_name == std_args[0])
        stamp = ec.current_time
        ec.write_data_line('hello')
        ec.wait_until(stamp + 0.02)
        ec.screen_prompt('test', 0.01, 0, None)
        ec.screen_prompt('test', 0.01, 0, ['1'])
        ec.screen_prompt(['test', 'ing'], 0.01, 0, ['1'])
        assert_raises(ValueError, ec.screen_prompt, 'foo', np.inf, 0, [])
        assert_raises(TypeError, ec.screen_prompt, 3, 0.01, 0, None)
        assert_equal(ec.wait_one_press(0.01), (None, None))
        assert_true(ec.wait_one_press(0.01, timestamp=False) is None)
        assert_equal(ec.wait_for_presses(0.01), [])
        assert_equal(ec.wait_for_presses(0.01, timestamp=False), [])
        assert_raises(ValueError, ec.get_presses)
        ec.listen_presses()
        assert_equal(ec.get_presses(), [])
        assert_equal(ec.get_presses(kind='presses'), [])
        assert_raises(ValueError, ec.get_presses, kind='foo')
        if this_rd == 'tdt':
            # TDT does not have key release events, so should raise an
            # exception if asked for them:
            assert_raises(RuntimeError, ec.get_presses, kind='releases')
            assert_raises(RuntimeError, ec.get_presses, kind='both')
        else:
            assert_equal(ec.get_presses(kind='both'), [])
            assert_equal(ec.get_presses(kind='releases'), [])
        ec.set_noise_db(0)
        ec.set_stim_db(20)
        # test buffer data handling
        ec.set_rms_checking(None)
        ec.load_buffer([0, 0, 0, 0, 0, 0])
        assert_raises(ValueError, ec.load_buffer, [0, 2, 0, 0, 0, 0])
        ec.load_buffer(np.zeros((100, )))
        ec.load_buffer(np.zeros((100, 1)))
        ec.load_buffer(np.zeros((100, 2)))
        ec.load_buffer(np.zeros((1, 100)))
        ec.load_buffer(np.zeros((2, 100)))
        data = np.empty(int(5e6), np.float32)  # too long for TDT
        if this_fs == get_tdt_rates()['25k']:
            assert_raises(RuntimeError, ec.load_buffer, data)
        else:
            ec.load_buffer(data)
        ec.load_buffer(np.zeros(2))
        del data
        assert_raises(ValueError, ec.stamp_triggers, 'foo')
        assert_raises(ValueError, ec.stamp_triggers, 0)
        assert_raises(ValueError, ec.stamp_triggers, 3)
        assert_raises(ValueError, ec.stamp_triggers, 1, check='foo')
        ec.stamp_triggers(3, check='int4')
        ec.stamp_triggers(2)
        ec.stamp_triggers([2, 4, 8])
        assert_raises(ValueError, ec.load_buffer, np.zeros((100, 3)))
        assert_raises(ValueError, ec.load_buffer, np.zeros((3, 100)))
        assert_raises(ValueError, ec.load_buffer, np.zeros((1, 1, 1)))

        # test RMS checking
        assert_raises(ValueError, ec.set_rms_checking, 'foo')
        # click: RMS 0.0135, should pass 'fullfile' and fail 'windowed'
        click = np.zeros((int(ec.fs / 4), ))  # 250 ms
        click[len(click) // 2] = 1.
        click[len(click) // 2 + 1] = -1.
        # noise: RMS 0.03, should fail both 'fullfile' and 'windowed'
        noise = np.random.normal(scale=0.03, size=(int(ec.fs / 4), ))
        ec.set_rms_checking(None)
        ec.load_buffer(click)  # should go unchecked
        ec.load_buffer(noise)  # should go unchecked
        ec.set_rms_checking('wholefile')
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter('always')
            ec.load_buffer(click)  # should pass
            assert_equal(len(w), 0)
            ec.load_buffer(noise)
            assert_equal(len(w), 1)
            ec.set_rms_checking('windowed')
            ec.load_buffer(click)
            assert_equal(len(w), 2)
            ec.load_buffer(noise)
            assert_equal(len(w), 3)

        ec.stop()
        ec.set_visible()
        ec.set_visible(False)
        ec.call_on_every_flip(partial(dummy_print, 'called start stimuli'))

        # Note: we put some wait_secs in here because otherwise the delay in
        # play start (e.g. for trigdel and onsetdel) can
        # mess things up! So we probably eventually should add
        # some safeguard against stopping too quickly after starting...

        #
        # First: identify_trial
        #
        noise = np.random.normal(scale=0.03, size=(int(ec.fs), ))
        ec.load_buffer(noise)
        assert_raises(RuntimeError, ec.start_stimulus)  # order violation
        assert_true(ec._playing is False)
        ec.start_stimulus(start_of_trial=False)  # should work
        ec.wait_secs(0.05)
        assert_true(ec._playing is True)
        assert_raises(RuntimeError, ec.trial_ok)  # order violation
        ec.stop()
        assert_true(ec._playing is False)
        # only binary for TTL
        assert_raises(KeyError, ec.identify_trial, ec_id='foo')  # need ttl_id
        assert_raises(TypeError, ec.identify_trial, ec_id='foo', ttl_id='bar')
        assert_raises(ValueError, ec.identify_trial, ec_id='foo', ttl_id=[2])
        assert_true(ec._playing is False)
        ec.identify_trial(ec_id='foo', ttl_id=[0, 1])
        assert_true(ec._playing is False)
        #
        # Second: start_stimuli
        #
        assert_raises(RuntimeError, ec.identify_trial, ec_id='foo', ttl_id=[0])
        assert_true(ec._playing is False)
        assert_raises(RuntimeError, ec.trial_ok)  # order violation
        assert_true(ec._playing is False)
        ec.start_stimulus(flip=False, when=-1)
        if ac != 'tdt':
            # dummy TDT version won't do this check properly, as
            # ec._ac._playing -> GetTagVal('playing') always gives False
            assert_raises(RuntimeError, ec.play)  # already played, must stop
        ec.wait_secs(0.05)
        ec.stop()
        assert_true(ec._playing is False)
        #
        # Third: trial_ok
        #
        assert_raises(RuntimeError, ec.start_stimulus)  # order violation
        assert_raises(RuntimeError, ec.identify_trial)  # order violation
        ec.trial_ok()
        # double-check
        assert_raises(RuntimeError, ec.start_stimulus)  # order violation
        ec.start_stimulus(start_of_trial=False)  # should work
        assert_raises(RuntimeError, ec.trial_ok)  # order violation
        ec.wait_secs(0.05)
        ec.stop()
        assert_true(ec._playing is False)

        ec.flip(-np.inf)
        assert_true(ec._playing is False)
        ec.estimate_screen_fs()
        assert_true(ec._playing is False)
        ec.play()
        ec.wait_secs(0.05)
        assert_true(ec._playing is True)
        ec.call_on_every_flip(None)
        ec.call_on_next_flip(ec.start_noise())
        ec.wait_secs(0.05)
        ec.stop()
        assert_true(ec._playing is False)
        ec.start_stimulus(start_of_trial=False)
        ec.call_on_next_flip(ec.stop_noise)
        ec.stop()
        ec.start_stimulus(start_of_trial=False)
        ec.get_mouse_position()
        ec.listen_clicks()
        ec.get_clicks()
        ec.toggle_cursor(False)
        ec.toggle_cursor(True, True)
        ec.wait_secs(0.001)
        print(ec.id_types)
        print(ec.stim_db)
        print(ec.noise_db)
        print(ec.on_next_flip_functions)
        print(ec.on_every_flip_functions)
        print(ec.window)
        data = ec.screenshot()
        assert_allclose(data.shape[:2], std_kwargs['window_size'])
        print(ec.fs)  # test fs support
        wait_secs(0.01)
        test_pix = (11.3, 0.5, 110003)
        print(test_pix)
        # test __repr__
        assert all([x in repr(ec) for x in ['foo', '"test"', '01']])
    del ec
Example #52
0
 def test_deprecation_warning(self):
     with warnings.catch_warnings(record=True) as warns:
         warnings.simplefilter('always')
         assign("contenttypes.change_contenttype", self.group)
         self.assertEqual(len(warns), 1)
         self.assertTrue(isinstance(warns[0].message, DeprecationWarning))
Example #53
0
 def test_error(self):
     with original_warnings.catch_warnings(module=self.module) as w:
         self.module.resetwarnings()
         self.module.filterwarnings("error", category=UserWarning)
         self.assertRaises(UserWarning, self.module.warn,
                           "FilterTests.test_error")
Example #54
0
 def test_summary(self):
     with warnings.catch_warnings():
         warnings.filterwarnings("ignore",
                                 "kurtosistest only valid for n>=20")
         self.model.fit().summary()
def concat_pandas(
    dfs, axis=0, join="outer", uniform=False, filter_warning=True, ignore_index=False
):
    if axis == 1:
        return pd.concat(dfs, axis=axis, join=join, sort=False)

    # Support concatenating indices along axis 0
    if isinstance(dfs[0], pd.Index):
        if isinstance(dfs[0], pd.CategoricalIndex):
            for i in range(1, len(dfs)):
                if not isinstance(dfs[i], pd.CategoricalIndex):
                    dfs[i] = dfs[i].astype("category")
            return pd.CategoricalIndex(union_categoricals(dfs), name=dfs[0].name)
        elif isinstance(dfs[0], pd.MultiIndex):
            first, rest = dfs[0], dfs[1:]
            if all(
                (isinstance(o, pd.MultiIndex) and o.nlevels >= first.nlevels)
                for o in rest
            ):
                arrays = [
                    concat([i._get_level_values(n) for i in dfs])
                    for n in range(first.nlevels)
                ]
                return pd.MultiIndex.from_arrays(arrays, names=first.names)

            to_concat = (first.values,) + tuple(k._values for k in rest)
            new_tuples = np.concatenate(to_concat)
            try:
                return pd.MultiIndex.from_tuples(new_tuples, names=first.names)
            except Exception:
                return pd.Index(new_tuples)
        return dfs[0].append(dfs[1:])

    # Handle categorical index separately
    dfs0_index = dfs[0].index

    has_categoricalindex = isinstance(dfs0_index, pd.CategoricalIndex) or (
        isinstance(dfs0_index, pd.MultiIndex)
        and any(isinstance(i, pd.CategoricalIndex) for i in dfs0_index.levels)
    )

    if has_categoricalindex:
        dfs2 = [df.reset_index(drop=True) for df in dfs]
        ind = concat([df.index for df in dfs])
    else:
        dfs2 = dfs
        ind = None

    # Concatenate the partitions together, handling categories as needed
    if (
        isinstance(dfs2[0], pd.DataFrame)
        if uniform
        else any(isinstance(df, pd.DataFrame) for df in dfs2)
    ):
        if uniform:
            dfs3 = dfs2
            cat_mask = dfs2[0].dtypes == "category"
        else:
            # When concatenating mixed dataframes and series on axis 1, Pandas
            # converts series to dataframes with a single column named 0, then
            # concatenates.
            dfs3 = [
                df
                if isinstance(df, pd.DataFrame)
                else df.to_frame().rename(columns={df.name: 0})
                for df in dfs2
            ]
            # pandas may raise a RuntimeWarning for comparing ints and strs
            with warnings.catch_warnings():
                warnings.simplefilter("ignore", RuntimeWarning)
                if filter_warning:
                    warnings.simplefilter("ignore", FutureWarning)
                cat_mask = pd.concat(
                    [(df.dtypes == "category").to_frame().T for df in dfs3],
                    join=join,
                    sort=False,
                ).any()

        if cat_mask.any():
            not_cat = cat_mask[~cat_mask].index
            # this should be aligned, so no need to filter warning
            out = pd.concat(
                [df[df.columns.intersection(not_cat)] for df in dfs3],
                join=join,
                sort=False,
            )
            temp_ind = out.index
            for col in cat_mask.index.difference(not_cat):
                # Find an example of categoricals in this column
                for df in dfs3:
                    sample = df.get(col)
                    if sample is not None:
                        break
                # Extract partitions, subbing in missing if needed
                parts = []
                for df in dfs3:
                    if col in df.columns:
                        parts.append(df[col])
                    else:
                        codes = np.full(len(df), -1, dtype="i8")
                        data = pd.Categorical.from_codes(
                            codes, sample.cat.categories, sample.cat.ordered
                        )
                        parts.append(data)
                out[col] = union_categoricals(parts)
                # Pandas resets index type on assignment if frame is empty
                # https://github.com/pandas-dev/pandas/issues/17101
                if not len(temp_ind):
                    out.index = temp_ind
            out = out.reindex(columns=cat_mask.index)
        else:
            # pandas may raise a RuntimeWarning for comparing ints and strs
            with warnings.catch_warnings():
                warnings.simplefilter("ignore", RuntimeWarning)
                if filter_warning:
                    warnings.simplefilter("ignore", FutureWarning)
                out = pd.concat(dfs3, join=join, sort=False)
    else:
        if is_categorical_dtype(dfs2[0].dtype):
            if ind is None:
                ind = concat([df.index for df in dfs2])
            return pd.Series(union_categoricals(dfs2), index=ind, name=dfs2[0].name)
        with warnings.catch_warnings():
            if filter_warning:
                warnings.simplefilter("ignore", FutureWarning)
            out = pd.concat(dfs2, join=join, sort=False)
    # Re-add the index if needed
    if ind is not None:
        out.index = ind
    return out
Example #56
0
def read_grayscale_u8(path_to_sequence: str) -> pims.FramesSequence:
    with warnings.catch_warnings():
        warnings.simplefilter('ignore')
        return _to_uint8(read_grayscale_f32(path_to_sequence))
Example #57
0
    def test_dt_accessor_api_for_categorical(self):
        # https://github.com/pandas-dev/pandas/issues/10661
        from pandas.core.indexes.accessors import Properties

        s_dr = Series(date_range("1/1/2015", periods=5, tz="MET"))
        c_dr = s_dr.astype("category")

        s_pr = Series(period_range("1/1/2015", freq="D", periods=5))
        c_pr = s_pr.astype("category")

        s_tdr = Series(timedelta_range("1 days", "10 days"))
        c_tdr = s_tdr.astype("category")

        # only testing field (like .day)
        # and bool (is_month_start)
        get_ops = lambda x: x._datetimelike_ops

        test_data = [
            ("Datetime", get_ops(DatetimeIndex), s_dr, c_dr),
            ("Period", get_ops(PeriodArray), s_pr, c_pr),
            ("Timedelta", get_ops(TimedeltaIndex), s_tdr, c_tdr),
        ]

        assert isinstance(c_dr.dt, Properties)

        special_func_defs = [
            ("strftime", ("%Y-%m-%d", ), {}),
            ("tz_convert", ("EST", ), {}),
            ("round", ("D", ), {}),
            ("floor", ("D", ), {}),
            ("ceil", ("D", ), {}),
            ("asfreq", ("D", ), {}),
            # FIXME: don't leave commented-out
            # ('tz_localize', ("UTC",), {}),
        ]
        _special_func_names = [f[0] for f in special_func_defs]

        # the series is already localized
        _ignore_names = ["tz_localize", "components"]

        for name, attr_names, s, c in test_data:
            func_names = [
                f for f in dir(s.dt)
                if not (f.startswith("_") or f in attr_names
                        or f in _special_func_names or f in _ignore_names)
            ]

            func_defs = [(f, (), {}) for f in func_names]
            for f_def in special_func_defs:
                if f_def[0] in dir(s.dt):
                    func_defs.append(f_def)

            for func, args, kwargs in func_defs:
                with warnings.catch_warnings():
                    if func == "to_period":
                        # dropping TZ
                        warnings.simplefilter("ignore", UserWarning)
                    res = getattr(c.dt, func)(*args, **kwargs)
                    exp = getattr(s.dt, func)(*args, **kwargs)

                tm.assert_equal(res, exp)

            for attr in attr_names:
                res = getattr(c.dt, attr)
                exp = getattr(s.dt, attr)

            if isinstance(res, DataFrame):
                tm.assert_frame_equal(res, exp)
            elif isinstance(res, Series):
                tm.assert_series_equal(res, exp)
            else:
                tm.assert_almost_equal(res, exp)

        invalid = Series([1, 2, 3]).astype("category")
        msg = "Can only use .dt accessor with datetimelike"

        with pytest.raises(AttributeError, match=msg):
            invalid.dt
        assert not hasattr(invalid, "str")
def assert_no_user_warning(f, args):
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter("always")
        f(*args)
        assert len(w) == 0
Example #59
-1
    def test_multiple(self):
        # test iterating over both entries and featurizers
        for iter_entries in [True, False]:
            multi_f = MultipleFeaturizer([self.single, self.multi],
                                         iterate_over_entries=iter_entries)
            data = self.make_test_data()

            self.assertArrayAlmostEqual([2, 0, 3], multi_f.featurize(1))

            self.assertArrayEqual(['A'], multi_f.citations())

            implementors = multi_f.implementors()
            self.assertIn('Us', implementors)
            self.assertIn('Them', implementors)
            self.assertEqual(2, len(implementors))

            # Ensure BaseFeaturizer operation without overriden featurize_dataframe
            with warnings.catch_warnings(record=True) as w:
                multi_f.featurize_dataframe(data, 'x')
                self.assertEqual(len(w), 0)
            self.assertArrayAlmostEqual(data['y'], [2, 3, 4])
            self.assertArrayAlmostEqual(data['w'], [0, 1, 2])
            self.assertArrayAlmostEqual(data['z'], [3, 4, 5])

            f = MatrixFeaturizer()
            multi_f = MultipleFeaturizer([self.single, self.multi, f])
            data = self.make_test_data()
            with warnings.catch_warnings(record=True) as w:
                multi_f.featurize_dataframe(data, 'x')
                self.assertEqual(len(w), 0)

            self.assertArrayAlmostEqual(data['representation'][0],
                                        [[1.0, 0.0], [0.0, 1.0]])
Example #60
-1
def test_tfidf_no_smoothing():
    X = [[1, 1, 1],
         [1, 1, 0],
         [1, 0, 0]]
    tr = TfidfTransformer(smooth_idf=False, norm='l2')
    tfidf = tr.fit_transform(X).toarray()
    assert_true((tfidf >= 0).all())

    # check normalization
    assert_array_almost_equal((tfidf ** 2).sum(axis=1), [1., 1., 1.])

    # the lack of smoothing make IDF fragile in the presence of feature with
    # only zeros
    X = [[1, 1, 0],
         [1, 1, 0],
         [1, 0, 0]]
    tr = TfidfTransformer(smooth_idf=False, norm='l2')

    # First we need to verify that numpy here provides div 0 warnings
    with warnings.catch_warnings(record=True) as w:
        1. / np.array([0.])
        numpy_provides_div0_warning = len(w) == 1

    with warnings.catch_warnings(record=True) as w:
        tfidf = tr.fit_transform(X).toarray()
        if not numpy_provides_div0_warning:
            raise SkipTest("Numpy does not provide div 0 warnings.")
        assert_equal(len(w), 1)
        # For Python 3 compatibility
        if hasattr(w[0].message, 'args'):
            assert_true("divide by zero" in w[0].message.args[0])
        else:
            assert_true("divide by zero" in w[0].message)