def testWarnings(self): # Smaller than the threshold: no warning. c_sparse = ops.IndexedSlices(array_ops.placeholder(dtypes.float32), array_ops.placeholder(dtypes.int32), constant([4, 4, 4, 4])) with warnings.catch_warnings(record=True) as w: math_ops.mul(c_sparse, 1.0) self.assertEqual(0, len(w)) # Greater than or equal to the threshold: warning. c_sparse = ops.IndexedSlices(array_ops.placeholder(dtypes.float32), array_ops.placeholder(dtypes.int32), constant([100, 100, 100, 100])) with warnings.catch_warnings(record=True) as w: math_ops.mul(c_sparse, 1.0) self.assertEqual(1, len(w)) self.assertTrue( "with 100000000 elements. This may consume a large amount of memory." in str(w[0].message)) # Unknown dense shape: warning. c_sparse = ops.IndexedSlices(array_ops.placeholder(dtypes.float32), array_ops.placeholder(dtypes.int32), array_ops.placeholder(dtypes.int32)) with warnings.catch_warnings(record=True) as w: math_ops.mul(c_sparse, 1.0) self.assertEqual(1, len(w)) self.assertTrue( "of unknown shape. This may consume a large amount of memory." in str(w[0].message))
def test_calculate_chpi_positions(): """Test calculation of cHPI positions """ trans, rot, t = head_pos_to_trans_rot_t(read_head_pos(pos_fname)) with warnings.catch_warnings(record=True): raw = Raw(chpi_fif_fname, allow_maxshield=True, preload=True) t -= raw.first_samp / raw.info['sfreq'] quats = _calculate_chpi_positions(raw, verbose='debug') trans_est, rot_est, t_est = head_pos_to_trans_rot_t(quats) _compare_positions((trans, rot, t), (trans_est, rot_est, t_est), 0.003) # degenerate conditions raw_no_chpi = Raw(test_fif_fname) assert_raises(RuntimeError, _calculate_chpi_positions, raw_no_chpi) raw_bad = raw.copy() for d in raw_bad.info['dig']: if d['kind'] == FIFF.FIFFV_POINT_HPI: d['coord_frame'] = 999 break assert_raises(RuntimeError, _calculate_chpi_positions, raw_bad) raw_bad = raw.copy() for d in raw_bad.info['dig']: if d['kind'] == FIFF.FIFFV_POINT_HPI: d['r'] = np.ones(3) raw_bad.crop(0, 1., copy=False) with warnings.catch_warnings(record=True): # bad pos with catch_logging() as log_file: _calculate_chpi_positions(raw_bad, verbose=True) # ignore HPI info header and [done] footer for line in log_file.getvalue().strip().split('\n')[4:-1]: assert_true('0/5 good' in line)
def test_long_cache_keys_shortened(self): cache_settings = { 'default': { 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', 'LOCATION': os.path.join(TOP_DIR, 'test.cache'), } } long_key_string = "X" * 251 with override_settings(CACHES=cache_settings): with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") # memcached limits key length to 250 cache.set(long_key_string, "hello cached world") self.assertEqual(len(w), 1) self.assertIsInstance(w[0].message, CacheKeyWarning) # Activate optional cache key length checker cache_settings['default']['KEY_FUNCTION'] = 'mainsite.utils.filter_cache_key' with override_settings(CACHES=cache_settings): with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") # memcached limits key length to 250 cache.set(long_key_string, "hello cached world") self.assertEqual(len(w), 0) retrieved = cache.get(long_key_string) self.assertEqual(retrieved, "hello cached world")
def test_iloc_getitem_multiindex(self): mi_labels = DataFrame(np.random.randn(4, 3), columns=[['i', 'i', 'j'], ['A', 'A', 'B']], index=[['i', 'i', 'j', 'k'], ['X', 'X', 'Y', 'Y']]) mi_int = DataFrame(np.random.randn(3, 3), columns=[[2, 2, 4], [6, 8, 10]], index=[[4, 4, 8], [8, 10, 12]]) # the first row rs = mi_int.iloc[0] with catch_warnings(record=True): xp = mi_int.ix[4].ix[8] tm.assert_series_equal(rs, xp, check_names=False) assert rs.name == (4, 8) assert xp.name == 8 # 2nd (last) columns rs = mi_int.iloc[:, 2] with catch_warnings(record=True): xp = mi_int.ix[:, 2] tm.assert_series_equal(rs, xp) # corner column rs = mi_int.iloc[2, 2] with catch_warnings(record=True): xp = mi_int.ix[:, 2].ix[2] assert rs == xp # this is basically regular indexing rs = mi_labels.iloc[2, 2] with catch_warnings(record=True): xp = mi_labels.ix['j'].ix[:, 'j'].ix[0, 0] assert rs == xp
def test_deprecated_score_func(): # test that old deprecated way of passing a score / loss function is still # supported X, y = make_classification(n_samples=200, n_features=100, random_state=0) clf = LinearSVC(random_state=0) cv = GridSearchCV(clf, {'C': [0.1, 1.0]}, scoring="f1") cv.fit(X[:180], y[:180]) y_pred = cv.predict(X[180:]) C = cv.best_estimator_.C clf = LinearSVC(random_state=0) cv = GridSearchCV(clf, {'C': [0.1, 1.0]}, score_func=f1_score) with warnings.catch_warnings(record=True): # catch deprecation warning cv.fit(X[:180], y[:180]) y_pred_func = cv.predict(X[180:]) C_func = cv.best_estimator_.C assert_array_equal(y_pred, y_pred_func) assert_equal(C, C_func) # test loss where greater is worse def f1_loss(y_true_, y_pred_): return -f1_score(y_true_, y_pred_) clf = LinearSVC(random_state=0) cv = GridSearchCV(clf, {'C': [0.1, 1.0]}, loss_func=f1_loss) with warnings.catch_warnings(record=True): # catch deprecation warning cv.fit(X[:180], y[:180]) y_pred_loss = cv.predict(X[180:]) C_loss = cv.best_estimator_.C assert_array_equal(y_pred, y_pred_loss) assert_equal(C, C_loss)
def test_deprecated_callbacks(self): # Tests that callback functions that return values are still supported but that warnings are generated def returns_cube(cube, field, filename): return cube def returns_no_cube(cube, field, filename): return iris.io.NO_CUBE fname = tests.get_data_path(["PP", "trui", "air_temp_init", "200812011200__qwqu12ff.initanl.pp"]) # Catch all warnings for returns_cube with warnings.catch_warnings(record=True) as generated_warnings_cube: warnings.simplefilter("always") r = iris.load(fname, callback=returns_cube) # Test that our warnings are present in the generated warnings: gen_warnings_cube = [str(x.message) for x in generated_warnings_cube] self.assertIn(iris.io.CALLBACK_DEPRECATION_MSG, gen_warnings_cube, "Callback deprecation warning message not issued.") # Catch all warnings for returns_no_cube with warnings.catch_warnings(record=True) as generated_warnings_no_cube: warnings.simplefilter("always") r = iris.load(fname, callback=returns_no_cube) # Test that our warnings are present in the generated warnings: gen_warnings_no_cube = [str(x.message) for x in generated_warnings_no_cube] self.assertIn(iris.io.CALLBACK_DEPRECATION_MSG, gen_warnings_no_cube, "Callback deprecation warning message not issued.")
def test_sparse_randomized_pca_inverse(): """Test that RandomizedPCA is inversible on sparse data""" rng = np.random.RandomState(0) n, p = 50, 3 X = rng.randn(n, p) # spherical data X[:, 1] *= 0.00001 # make middle component relatively small # no large means because the sparse version of randomized pca does not do # centering to avoid breaking the sparsity X = csr_matrix(X) # same check that we can find the original data from the transformed signal # (since the data is almost of rank n_components) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always", DeprecationWarning) pca = RandomizedPCA(n_components=2, random_state=0).fit(X) assert_equal(len(w), 1) assert_equal(w[0].category, DeprecationWarning) Y = pca.transform(X) Y_inverse = pca.inverse_transform(Y) assert_almost_equal(X.todense(), Y_inverse, decimal=2) # same as above with whitening (approximate reconstruction) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always", DeprecationWarning) pca = RandomizedPCA(n_components=2, whiten=True, random_state=0).fit(X) assert_equal(len(w), 1) assert_equal(w[0].category, DeprecationWarning) Y = pca.transform(X) Y_inverse = pca.inverse_transform(Y) relative_max_delta = (np.abs(X.todense() - Y_inverse) / np.abs(X).mean()).max() # XXX: this does not seam to work as expected: assert_almost_equal(relative_max_delta, 0.91, decimal=2)
def run_tests(df, rhs, right): # label, index, slice r, i, s = list('bcd'), [1, 2, 3], slice(1, 4) c, j, l = ['joe', 'jolie'], [1, 2], slice(1, 3) left = df.copy() left.loc[r, c] = rhs tm.assert_frame_equal(left, right) left = df.copy() left.iloc[i, j] = rhs tm.assert_frame_equal(left, right) left = df.copy() with catch_warnings(record=True): left.ix[s, l] = rhs tm.assert_frame_equal(left, right) left = df.copy() with catch_warnings(record=True): left.ix[i, j] = rhs tm.assert_frame_equal(left, right) left = df.copy() with catch_warnings(record=True): left.ix[r, c] = rhs tm.assert_frame_equal(left, right)
def check_classifiers_input_shapes(name, Classifier): iris = load_iris() X, y = iris.data, iris.target X, y = shuffle(X, y, random_state=1) X = StandardScaler().fit_transform(X) # catch deprecation warnings with warnings.catch_warnings(record=True): classifier = Classifier() set_fast_parameters(classifier) set_random_state(classifier) # fit classifier.fit(X, y) y_pred = classifier.predict(X) set_random_state(classifier) # Check that when a 2D y is given, a DataConversionWarning is # raised with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always", DataConversionWarning) warnings.simplefilter("ignore", RuntimeWarning) classifier.fit(X, y[:, np.newaxis]) msg = "expected 1 DataConversionWarning, got: %s" % ( ", ".join([str(w_x) for w_x in w])) assert_equal(len(w), 1, msg) assert_array_equal(y_pred, classifier.predict(X))
def test_get_url_shortener(self): us_settings.URL_SHORTENER_BACKEND = 'mymodule.myclass' try: with warnings.catch_warnings(record=True) as w: self.assertEquals(get_url_shortener(), default_backend) self.assertTrue(issubclass(w[-1].metatype, RuntimeWarning)) self.assertEquals( str(w[-1].message), 'mymodule.myclass backend cannot be imported') except AttributeError: # Fail under Python2.5, because of'warnings.catch_warnings' pass us_settings.URL_SHORTENER_BACKEND = 'gstudio.tests.custom_url_shortener' try: with warnings.catch_warnings(record=True) as w: self.assertEquals(get_url_shortener(), default_backend) self.assertTrue(issubclass(w[-1].metatype, RuntimeWarning)) self.assertEquals( str(w[-1].message), 'This backend only exists for testing') except AttributeError: # Fail under Python2.5, because of'warnings.catch_warnings' pass us_settings.URL_SHORTENER_BACKEND = 'gstudio.url_shortener'\ '.backends.default' self.assertEquals(get_url_shortener(), default_backend)
def setUp(self): self.aln_file = [TEST_ALIGN_FILE1, TEST_ALIGN_FILE2, TEST_ALIGN_FILE3, TEST_ALIGN_FILE4, TEST_ALIGN_FILE5, TEST_ALIGN_FILE6] alns = [] for i in self.aln_file: if i[1] == 'parse': nucl = SeqIO.parse(i[0][0], 'fasta', alphabet=IUPAC.IUPACUnambiguousDNA()) prot = AlignIO.read(i[0][1], 'clustal', alphabet=IUPAC.protein) with warnings.catch_warnings(): warnings.simplefilter('ignore') caln = codonalign.build(prot, nucl, alphabet=codonalign.default_codon_alphabet) elif i[1] == 'index': nucl = SeqIO.index(i[0][0], 'fasta', alphabet=IUPAC.IUPACUnambiguousDNA()) prot = AlignIO.read(i[0][1], 'clustal', alphabet=IUPAC.protein) with warnings.catch_warnings(): warnings.simplefilter('ignore') caln = codonalign.build(prot, nucl, alphabet=codonalign.default_codon_alphabet, max_score=20) elif i[1] == 'id': nucl = SeqIO.parse(i[0][0], 'fasta', alphabet=IUPAC.IUPACUnambiguousDNA()) prot = AlignIO.read(i[0][1], 'clustal', alphabet=IUPAC.protein) with open(i[0][2]) as handle: id = dict((i.split()[0], i.split()[1]) for i in handle) with warnings.catch_warnings(): warnings.simplefilter('ignore') caln = codonalign.build(prot, nucl, corr_dict=id, alphabet=codonalign.default_codon_alphabet) alns.append(caln) nucl.close() # Close the indexed FASTA file self.alns = alns
def check_clustering(name, Alg): X, y = make_blobs(n_samples=50, random_state=1) X, y = shuffle(X, y, random_state=7) X = StandardScaler().fit_transform(X) n_samples, n_features = X.shape # catch deprecation and neighbors warnings with warnings.catch_warnings(record=True): alg = Alg() set_fast_parameters(alg) if hasattr(alg, "n_clusters"): alg.set_params(n_clusters=3) set_random_state(alg) if name == 'AffinityPropagation': alg.set_params(preference=-100) alg.set_params(max_iter=100) # fit alg.fit(X) # with lists alg.fit(X.tolist()) assert_equal(alg.labels_.shape, (n_samples,)) pred = alg.labels_ assert_greater(adjusted_rand_score(pred, y), 0.4) # fit another time with ``fit_predict`` and compare results if name is 'SpectralClustering': # there is no way to make Spectral clustering deterministic :( return set_random_state(alg) with warnings.catch_warnings(record=True): pred2 = alg.fit_predict(X) assert_array_equal(pred, pred2)
def test_all(): """Test maxwell filter using all options""" raw_fnames = (raw_fname, raw_fname, erm_fname, sample_fname) sss_fnames = (sss_st1FineCalCrossTalkRegIn_fname, sss_st1FineCalCrossTalkRegInTransSample_fname, sss_erm_st1FineCalCrossTalkRegIn_fname, sss_samp_fname) fine_cals = (fine_cal_fname, fine_cal_fname, fine_cal_fname, fine_cal_mgh_fname) coord_frames = ('head', 'head', 'meg', 'head') ctcs = (ctc_fname, ctc_fname, ctc_fname, ctc_mgh_fname) mins = (3.5, 3.5, 1.2, 0.9) meds = (10.9, 10.4, 3.2, 6.) st_durs = (1., 1., 1., None) destinations = (None, sample_fname, None, None) origins = (mf_head_origin, mf_head_origin, mf_meg_origin, mf_head_origin) for ii, rf in enumerate(raw_fnames): with warnings.catch_warnings(record=True): # maxshield raw = Raw(rf, allow_maxshield=True).crop(0., 1., copy=False) with warnings.catch_warnings(record=True): # head fit off-center sss_py = maxwell_filter( raw, calibration=fine_cals[ii], cross_talk=ctcs[ii], st_duration=st_durs[ii], coord_frame=coord_frames[ii], destination=destinations[ii], origin=origins[ii]) sss_mf = Raw(sss_fnames[ii]) assert_meg_snr(sss_py, sss_mf, mins[ii], meds[ii], msg=rf)
def test_read_chunks_115(self): files_115 = [self.dta2_115, self.dta3_115, self.dta4_115, self.dta14_115, self.dta15_115, self.dta16_115, self.dta17_115, self.dta18_115, self.dta19_115, self.dta20_115] for fname in files_115: for chunksize in 1,2: for convert_categoricals in False, True: for convert_dates in False, True: # Read the whole file with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") parsed = read_stata(fname, convert_categoricals=convert_categoricals, convert_dates=convert_dates) # Compare to what we get when reading by chunk itr = read_stata(fname, iterator=True, convert_dates=convert_dates, convert_categoricals=convert_categoricals) pos = 0 for j in range(5): with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") try: chunk = itr.read(chunksize) except StopIteration: break from_frame = parsed.iloc[pos:pos+chunksize, :] tm.assert_frame_equal(from_frame, chunk, check_dtype=False, check_datetimelike_compat=True) pos += chunksize
def run_tests(df, rhs, right): # label, index, slice lbl_one, idx_one, slice_one = list('bcd'), [1, 2, 3], slice(1, 4) lbl_two, idx_two, slice_two = ['joe', 'jolie'], [1, 2], slice(1, 3) left = df.copy() left.loc[lbl_one, lbl_two] = rhs tm.assert_frame_equal(left, right) left = df.copy() left.iloc[idx_one, idx_two] = rhs tm.assert_frame_equal(left, right) left = df.copy() with catch_warnings(record=True): # XXX: finer-filter here. simplefilter("ignore") left.ix[slice_one, slice_two] = rhs tm.assert_frame_equal(left, right) left = df.copy() with catch_warnings(record=True): simplefilter("ignore") left.ix[idx_one, idx_two] = rhs tm.assert_frame_equal(left, right) left = df.copy() with catch_warnings(record=True): simplefilter("ignore") left.ix[lbl_one, lbl_two] = rhs tm.assert_frame_equal(left, right)
def test_head_translation(): """Test Maxwell filter head translation""" with warnings.catch_warnings(record=True): # maxshield raw = Raw(raw_fname, allow_maxshield=True).crop(0., 1., False) # First try with an unchanged destination raw_sss = maxwell_filter(raw, destination=raw_fname, origin=mf_head_origin, regularize=None, bad_condition='ignore') assert_meg_snr(raw_sss, Raw(sss_std_fname).crop(0., 1., False), 200.) # Now with default with warnings.catch_warnings(record=True): with catch_logging() as log: raw_sss = maxwell_filter(raw, destination=mf_head_origin, origin=mf_head_origin, regularize=None, bad_condition='ignore', verbose='warning') assert_true('over 25 mm' in log.getvalue()) assert_meg_snr(raw_sss, Raw(sss_trans_default_fname), 125.) destination = np.eye(4) destination[2, 3] = 0.04 assert_allclose(raw_sss.info['dev_head_t']['trans'], destination) # Now to sample's head pos with warnings.catch_warnings(record=True): with catch_logging() as log: raw_sss = maxwell_filter(raw, destination=sample_fname, origin=mf_head_origin, regularize=None, bad_condition='ignore', verbose='warning') assert_true('= 25.6 mm' in log.getvalue()) assert_meg_snr(raw_sss, Raw(sss_trans_sample_fname), 350.) assert_allclose(raw_sss.info['dev_head_t']['trans'], read_info(sample_fname)['dev_head_t']['trans']) # Degenerate cases assert_raises(RuntimeError, maxwell_filter, raw, destination=mf_head_origin, coord_frame='meg') assert_raises(ValueError, maxwell_filter, raw, destination=[0.] * 4)
def test_random_pair(self) : self.assertRaises(ValueError, dedupe.core.randomPairs, 1, 10) assert dedupe.core.randomPairs(10, 10).any() random.seed(123) numpy.random.seed(123) random_pairs = dedupe.core.randomPairs(10, 5) assert numpy.array_equal(random_pairs, numpy.array([[ 0, 3], [ 3, 8], [ 4, 9], [ 5, 9], [ 2, 3]])) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") dedupe.core.randomPairs(10, 10**6) assert len(w) == 1 assert str(w[-1].message) == "Requested sample of size 1000000, only returning 45 possible pairs" with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") dedupe.core.randomPairs(10**40, 10) assert len(w) == 2 assert str(w[0].message) == "There may be duplicates in the sample" assert "Asked to sample pairs from" in str(w[1].message) random.seed(123) numpy.random.seed(123) assert numpy.array_equal(dedupe.core.randomPairs(11**9, 1), numpy.array([[1228959102, 1840268610]]))
def test_make_morph_maps(): """Test reading and creating morph maps.""" # make a new fake subjects_dir tempdir = _TempDir() for subject in ('sample', 'sample_ds', 'fsaverage_ds'): os.mkdir(op.join(tempdir, subject)) os.mkdir(op.join(tempdir, subject, 'surf')) regs = ('reg', 'left_right') if subject == 'fsaverage_ds' else ('reg',) for hemi in ['lh', 'rh']: for reg in regs: args = [subject, 'surf', hemi + '.sphere.' + reg] copyfile(op.join(subjects_dir, *args), op.join(tempdir, *args)) for subject_from, subject_to, xhemi in ( ('fsaverage_ds', 'sample_ds', False), ('fsaverage_ds', 'fsaverage_ds', True)): # trigger the creation of morph-maps dir and create the map with warnings.catch_warnings(record=True): mmap = read_morph_map(subject_from, subject_to, tempdir, xhemi=xhemi) mmap2 = read_morph_map(subject_from, subject_to, subjects_dir, xhemi=xhemi) assert_equal(len(mmap), len(mmap2)) for m1, m2 in zip(mmap, mmap2): # deal with sparse matrix stuff diff = (m1 - m2).data assert_allclose(diff, np.zeros_like(diff), atol=1e-3, rtol=0) # This will also trigger creation, but it's trivial with warnings.catch_warnings(record=True): mmap = read_morph_map('sample', 'sample', subjects_dir=tempdir) for mm in mmap: assert_true((mm - sparse.eye(mm.shape[0], mm.shape[0])).sum() == 0)
def test_array_richcompare_legacy_weirdness(self): # It doesn't really work to use assert_deprecated here, b/c part of # the point of assert_deprecated is to check that when warnings are # set to "error" mode then the error is propagated -- which is good! # But here we are testing a bunch of code that is deprecated *because* # it has the habit of swallowing up errors and converting them into # different warnings. So assert_warns will have to be sufficient. assert_warns(FutureWarning, lambda: np.arange(2) == "a") assert_warns(FutureWarning, lambda: np.arange(2) != "a") # No warning for scalar comparisons with warnings.catch_warnings(): warnings.filterwarnings("error") assert_(not (np.array(0) == "a")) assert_(np.array(0) != "a") assert_(not (np.int16(0) == "a")) assert_(np.int16(0) != "a") for arg1 in [np.asarray(0), np.int16(0)]: struct = np.zeros(2, dtype="i4,i4") for arg2 in [struct, "a"]: for f in [operator.lt, operator.le, operator.gt, operator.ge]: if sys.version_info[0] >= 3: # py3 with warnings.catch_warnings() as l: warnings.filterwarnings("always") assert_raises(TypeError, f, arg1, arg2) assert_(not l) else: # py2 assert_warns(DeprecationWarning, f, arg1, arg2)
def test_random_pair_match(self) : self.assertRaises(ValueError, dedupe.core.randomPairsMatch, 1, 0, 10) self.assertRaises(ValueError, dedupe.core.randomPairsMatch, 0, 0, 10) self.assertRaises(ValueError, dedupe.core.randomPairsMatch, 0, 1, 10) assert len(dedupe.core.randomPairsMatch(100, 100, 100)) == 100 assert len(dedupe.core.randomPairsMatch(10, 10, 99)) == 99 with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") pairs = dedupe.core.randomPairsMatch(10, 10, 200) assert len(w) == 1 assert str(w[-1].message) == "Requested sample of size 200, only returning 100 possible pairs" assert len(pairs) == 100 with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") pairs = dedupe.core.randomPairsMatch(10, 10, 200) assert len(w) == 1 assert str(w[-1].message) == "Requested sample of size 200, only returning 100 possible pairs" random.seed(123) numpy.random.seed(123) pairs = dedupe.core.randomPairsMatch(10, 10, 10) assert pairs == set([(7, 3), (3, 3), (2, 9), (6, 0), (2, 0), (1, 9), (9, 4), (0, 4), (1, 0), (1, 1)])
def test_misspecifications(): # Tests for model specification and misspecification exceptions endog = np.arange(20).reshape(10,2) # Bad trend specification assert_raises(ValueError, varmax.VARMAX, endog, order=(1,0), trend='') # Bad error_cov_type specification assert_raises(ValueError, varmax.VARMAX, endog, order=(1,0), error_cov_type='') # Bad order specification assert_raises(ValueError, varmax.VARMAX, endog, order=(0,0)) with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') varmax.VARMAX(endog, order=(1,1)) # Warning with VARMA specification with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') varmax.VARMAX(endog, order=(1,1)) message = ('Estimation of VARMA(p,q) models is not generically robust,' ' due especially to identification issues.') assert_equal(str(w[0].message), message) warnings.resetwarnings()
def test_class_weight_classifiers(): # test that class_weight works and that the semantics are consistent classifiers = all_estimators(type_filter="classifier") with warnings.catch_warnings(record=True): classifiers = [c for c in classifiers if "class_weight" in c[1]().get_params().keys()] for n_centers in [2, 3]: # create a very noisy dataset X, y = make_blobs(centers=n_centers, random_state=0, cluster_std=20) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0) for name, Classifier in classifiers: if name == "NuSVC": # the sparse version has a parameter that doesn't do anything continue if name.endswith("NB"): # NaiveBayes classifiers have a somewhat different interface. # FIXME SOON! continue if n_centers == 2: class_weight = {0: 1000, 1: 0.0001} else: class_weight = {0: 1000, 1: 0.0001, 2: 0.0001} with warnings.catch_warnings(record=True): classifier = Classifier(class_weight=class_weight) if hasattr(classifier, "n_iter"): classifier.set_params(n_iter=100) set_random_state(classifier) classifier.fit(X_train, y_train) y_pred = classifier.predict(X_test) assert_greater(np.mean(y_pred == 0), 0.9)
def _check_predict_proba(clf, X, y): proba = clf.predict_proba(X) with warnings.catch_warnings(): warnings.simplefilter("ignore") # We know that we can have division by zero log_proba = clf.predict_log_proba(X) y = np.atleast_1d(y) if y.ndim == 1: y = np.reshape(y, (-1, 1)) n_outputs = y.shape[1] n_samples = len(X) if n_outputs == 1: proba = [proba] log_proba = [log_proba] for k in xrange(n_outputs): assert_equal(proba[k].shape[0], n_samples) assert_equal(proba[k].shape[1], len(np.unique(y[:, k]))) assert_array_equal(proba[k].sum(axis=1), np.ones(len(X))) with warnings.catch_warnings(): warnings.simplefilter("ignore") # We know that we can have division by zero assert_array_equal(np.log(proba[k]), log_proba[k])
def test_popen(self): mswindows = (sys.platform == "win32") if mswindows: command = '"{}" -c "print(\'Hello\')"'.format(sys.executable) else: command = "'{}' -c 'print(\"Hello\")'".format(sys.executable) with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) with platform.popen(command) as stdout: hello = stdout.read().strip() stdout.close() self.assertEqual(hello, "Hello") data = 'plop' if mswindows: command = '"{}" -c "import sys; data=sys.stdin.read(); exit(len(data))"' else: command = "'{}' -c 'import sys; data=sys.stdin.read(); exit(len(data))'" command = command.format(sys.executable) with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) with platform.popen(command, 'w') as stdin: stdout = stdin.write(data) ret = stdin.close() self.assertIsNotNone(ret) if os.name == 'nt': returncode = ret else: returncode = ret >> 8 self.assertEqual(returncode, len(data))
def test_graph_iterative(self): graph = MigrationGraph() root = ("app_a", "1") graph.add_node(root, None) expected = [root] for i in range(2, 1000): parent = ("app_a", str(i - 1)) child = ("app_a", str(i)) graph.add_node(child, None) graph.add_dependency(str(i), child, parent) expected.append(child) leaf = expected[-1] with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always', RuntimeWarning) forwards_plan = graph.forwards_plan(leaf) self.assertEqual(len(w), 1) self.assertTrue(issubclass(w[-1].category, RuntimeWarning)) self.assertEqual(str(w[-1].message), RECURSION_DEPTH_WARNING) self.assertEqual(expected, forwards_plan) with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always', RuntimeWarning) backwards_plan = graph.backwards_plan(root) self.assertEqual(len(w), 1) self.assertTrue(issubclass(w[-1].category, RuntimeWarning)) self.assertEqual(str(w[-1].message), RECURSION_DEPTH_WARNING) self.assertEqual(expected[::-1], backwards_plan)
def _check_roundtrip(self, frame): _skip_if_no_MySQLdb() drop_sql = "DROP TABLE IF EXISTS test_table" cur = self.db.cursor() with warnings.catch_warnings(): warnings.filterwarnings("ignore", "Unknown table.*") cur.execute(drop_sql) sql.write_frame(frame, name='test_table', con=self.db, flavor='mysql') result = sql.read_frame("select * from test_table", self.db) # HACK! Change this once indexes are handled properly. result.index = frame.index result.index.name = frame.index.name expected = frame tm.assert_frame_equal(result, expected) frame['txt'] = ['a'] * len(frame) frame2 = frame.copy() index = Index(lrange(len(frame2))) + 10 frame2['Idx'] = index drop_sql = "DROP TABLE IF EXISTS test_table2" cur = self.db.cursor() with warnings.catch_warnings(): warnings.filterwarnings("ignore", "Unknown table.*") cur.execute(drop_sql) sql.write_frame(frame2, name='test_table2', con=self.db, flavor='mysql') result = sql.read_frame("select * from test_table2", self.db, index_col='Idx') expected = frame.copy() # HACK! Change this once indexes are handled properly. expected.index = index expected.index.names = result.index.names tm.assert_frame_equal(expected, result)
def test_evoked_io_from_epochs(): """Test IO of evoked data made from epochs """ # offset our tmin so we don't get exactly a zero value when decimating with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") epochs = Epochs(raw, events[:4], event_id, tmin + 0.011, tmax, picks=picks, baseline=(None, 0), decim=5) assert_true(len(w) == 1) evoked = epochs.average() evoked.save(op.join(tempdir, "evoked-ave.fif")) evoked2 = read_evokeds(op.join(tempdir, "evoked-ave.fif"))[0] assert_allclose(evoked.data, evoked2.data, rtol=1e-4, atol=1e-20) assert_allclose(evoked.times, evoked2.times, rtol=1e-4, atol=1 / evoked.info["sfreq"]) # now let's do one with negative time with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") epochs = Epochs(raw, events[:4], event_id, 0.1, tmax, picks=picks, baseline=(0.1, 0.2), decim=5) evoked = epochs.average() evoked.save(op.join(tempdir, "evoked-ave.fif")) evoked2 = read_evokeds(op.join(tempdir, "evoked-ave.fif"))[0] assert_allclose(evoked.data, evoked2.data, rtol=1e-4, atol=1e-20) assert_allclose(evoked.times, evoked2.times, rtol=1e-4, atol=1e-20) # should be equivalent to a cropped original with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") epochs = Epochs(raw, events[:4], event_id, -0.2, tmax, picks=picks, baseline=(0.1, 0.2), decim=5) evoked = epochs.average() evoked.crop(0.099, None) assert_allclose(evoked.data, evoked2.data, rtol=1e-4, atol=1e-20) assert_allclose(evoked.times, evoked2.times, rtol=1e-4, atol=1e-20)
def test_read_epochs_bad_events(): """Test epochs when events are at the beginning or the end of the file """ # Event at the beginning epochs = Epochs( raw, np.array([[raw.first_samp, 0, event_id]]), event_id, tmin, tmax, picks=picks, baseline=(None, 0) ) with warnings.catch_warnings(record=True): evoked = epochs.average() epochs = Epochs( raw, np.array([[raw.first_samp, 0, event_id]]), event_id, tmin, tmax, picks=picks, baseline=(None, 0) ) epochs.drop_bad_epochs() with warnings.catch_warnings(record=True): evoked = epochs.average() # Event at the end epochs = Epochs( raw, np.array([[raw.last_samp, 0, event_id]]), event_id, tmin, tmax, picks=picks, baseline=(None, 0) ) with warnings.catch_warnings(record=True): evoked = epochs.average() assert evoked warnings.resetwarnings()
def read_rgb_f32(path_to_sequence: str) -> pims.FramesSequence: with warnings.catch_warnings(): warnings.simplefilter('ignore') return _to_float32(pims.open(path_to_sequence))
def _runCompatible(self, compatibleList): """Calculate the metric values for set of (parent and child) bundles, as well as the summary stats, and write to disk. Parameters ----------- compatibleList : list List of dictionary keys, of the metricBundles which can be calculated together. This means they are 'compatible' and have the same slicer, constraint, and non-conflicting mappers and stackers. """ if self.verbose: print('Running metrics %s' % compatibleList) bDict = self.bundleDict # {key: self.bundleDict.get(key) for key in compatibleList} # Find the unique stackers and maps. These are already "compatible" (as id'd by compatibleList). uniqStackers = [] allStackers = [] uniqMaps = [] allMaps = [] for b in bDict.values(): allStackers += b.stackerList allMaps += b.mapsList for s in allStackers: if s not in uniqStackers: uniqStackers.append(s) for m in allMaps: if m not in uniqMaps: uniqMaps.append(m) if len(uniqMaps) > 0: print("Got some maps .. that was unexpected at the moment. Can't use them here yet.") # Set up all of the metric values, including for the child bundles. for k in compatibleList: b = self.bundleDict[k] b._setupMetricValues() for cb in b.childBundles.values(): cb._setupMetricValues() # Calculate the metric values. for i, slicePoint in enumerate(self.slicer): ssoObs = slicePoint['obs'] for j, Hval in enumerate(slicePoint['Hvals']): # Run stackers to add extra columns (that depend on Hval) with warnings.catch_warnings(): warnings.simplefilter('ignore') for s in uniqStackers: ssoObs = s.run(ssoObs, slicePoint['orbit']['H'], Hval) # Run all the parent metrics. for k in compatibleList: b = self.bundleDict[k] # Mask the parent metric (and then child metrics) if there was no data. if len(ssoObs) == 0: b.metricValues.mask[i][j] = True for cb in list(b.childBundles.values()): cb.metricValues.mask[i][j] = True # Otherwise, calculate the metric value for the parent, and then child. else: # Calculate for the parent. mVal = b.metric.run(ssoObs, slicePoint['orbit'], Hval) # Mask if the parent metric returned a bad value. if mVal == b.metric.badval: b.metricValues.mask[i][j] = True for cb in b.childBundles.values(): cb.metricValues.mask[i][j] = True # Otherwise, set the parent value and calculate the child metric values as well. else: b.metricValues.data[i][j] = mVal for cb in b.childBundles.values(): childVal = cb.metric.run(ssoObs, slicePoint['orbit'], Hval, mVal) if childVal == cb.metric.badval: cb.metricValues.mask[i][j] = True else: cb.metricValues.data[i][j] = childVal for k in compatibleList: b = self.bundleDict[k] b.computeSummaryStats(self.resultsDb) for cB in b.childBundles.values(): cB.computeSummaryStats(self.resultsDb) # Write to disk. cB.write(outDir=self.outDir, resultsDb=self.resultsDb) # Write to disk. b.write(outDir=self.outDir, resultsDb=self.resultsDb)
def soft_thresholding_vector(a, lamda): """Soft-thresholding for vectors.""" with warnings.catch_warnings(): warnings.simplefilter("ignore") return np.maximum(1 - lamda / np.linalg.norm(a), 0) * a
def main(night_name=None, files=None): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) p = spirouStartup.LoadArguments(p, night_name, files, mainfitsdir='reduced') p = spirouStartup.InitialFileSetup(p, calibdb=True) # set up function name main_name = __NAME__ + '.main()' # ---------------------------------------------------------------------- # Load first file # ---------------------------------------------------------------------- loc = ParamDict() rd = spirouImage.ReadImage(p, p['FITSFILENAME']) loc['DATA'], loc['DATAHDR'], loc['XDIM'], loc['YDIM'] = rd loc.set_sources(['DATA', 'DATAHDR', 'XDIM', 'YDIM'], main_name) # ---------------------------------------------------------------------- # Get object name, airmass and berv # ---------------------------------------------------------------------- # Get object name loc['OBJNAME'] = spirouImage.GetObjName(p, loc['DATAHDR']) # Get the airmass loc['AIRMASS'] = spirouImage.GetAirmass(p, loc['DATAHDR']) # Get the Barycentric correction from header p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, loc['DATAHDR']) # set sources source = main_name + '+ spirouImage.ReadParams()' loc.set_sources(['OBJNAME', 'AIRMASS'], source) loc.set_sources(['OBJNAME', 'AIRMASS'], source) # ---------------------------------------------------------------------- # Read wavelength solution # ---------------------------------------------------------------------- # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # used for plotting only wout = spirouImage.GetWaveSolution(p, image=loc['DATA'], hdr=loc['DATAHDR'], return_wavemap=True, fiber=wave_fiber) _, loc['WAVE'], _ = wout loc.set_source('WAVE', main_name) # ---------------------------------------------------------------------- # Get and Normalise the blaze # ---------------------------------------------------------------------- p, loc = spirouTelluric.GetNormalizedBlaze(p, loc, loc['DATAHDR']) # ---------------------------------------------------------------------- # Load transmission files # ---------------------------------------------------------------------- transdata = spirouDB.GetDatabaseTellMap(p) trans_files = transdata[0] # make sure we have unique filenames for trans_files trans_files = np.unique(trans_files) # ---------------------------------------------------------------------- # Start plotting # ---------------------------------------------------------------------- if p['DRS_PLOT'] > 0: # start interactive plot sPlt.start_interactive_session(p) # ---------------------------------------------------------------------- # Load template (if available) # ---------------------------------------------------------------------- # read filename from telluDB template_file = spirouDB.GetDatabaseObjTemp(p, loc['OBJNAME'], required=False) # if we don't have a template flag it if template_file is None: loc['FLAG_TEMPLATE'] = False loc['TEMPLATE'] = None else: loc['FLAG_TEMPLATE'] = True # load template template, _, _, _ = spirouImage.ReadImage(p, template_file) # add to loc loc['TEMPLATE'] = template # set the source for flag and template loc.set_sources(['FLAG_TEMPLATE', 'TEMPLATE'], main_name) # ---------------------------------------------------------------------- # load the expected atmospheric transmission # ---------------------------------------------------------------------- # read filename from telluDB tapas_file_names = spirouDB.GetDatabaseTellConv(p) tapas_file_name = tapas_file_names[-1] # load atmospheric transmission loc['TAPAS_ALL_SPECIES'] = np.load(tapas_file_name) loc.set_source('TAPAS_ALL_SPECIES', main_name) # ---------------------------------------------------------------------- # Generate the absorption map # ---------------------------------------------------------------------- # get number of files nfiles = len(trans_files) npc = p['TELLU_NUMBER_OF_PRINCIPLE_COMP'] # check that we have enough files (greater than number of principle # components) if nfiles <= npc: emsg1 = 'Not enough "TELL_MAP" files in telluDB to run PCA analysis' emsg2 = '\tNumber of files = {0}, number of PCA components = {1}' emsg3 = '\tNumber of files > number of PCA components' emsg4 = '\tAdd more files or reduce number of PCA components' WLOG(p, 'error', [emsg1, emsg2.format(nfiles, npc), emsg3, emsg4]) # check whether we can used pre-saved abso filetime = spirouImage.GetMostRecent(trans_files) tout = spirouConfig.Constants.TELLU_ABSO_SAVE(p, filetime) abso_save_file, absoprefix = tout use_saved = os.path.exists(abso_save_file) # noinspection PyBroadException try: # try loading from file abso = np.load(abso_save_file) # log progress wmsg = 'Loaded abso from file {0}'.format(abso_save_file) WLOG(p, '', wmsg) except: # set up storage for the absorption abso = np.zeros([nfiles, np.product(loc['DATA'].shape)]) # loop around outputfiles and add them to abso for it, filename in enumerate(trans_files): # load data data_it, _, _, _ = spirouImage.ReadImage(p, filename=filename) # push data into array abso[it, :] = data_it.reshape(np.product(loc['DATA'].shape)) # log progres wmsg = 'Saving abso to file {0}'.format(abso_save_file) WLOG(p, '', wmsg) # remove all abso save files (only need most recent one) afolder = os.path.dirname(abso_save_file) afilelist = os.listdir(afolder) for afile in afilelist: if afile.startswith(absoprefix): os.remove(os.path.join(afolder, afile)) # save to file for later use np.save(abso_save_file, abso) # filter out rows of all NaNs # TODO: Why are we getting all NaN e2ds files? abso_filtered = [] for row in range(len(abso)): if np.sum(np.isnan(abso[row])) != len(abso[row]): abso_filtered.append(abso[row]) else: wargs = [trans_files[row]] WLOG(p, '', 'Removing trans file {0} (all NaN)'.format(*wargs)) abso = np.array(abso_filtered) # log the absorption cube with warnings.catch_warnings(record=True) as w: log_abso = np.log(abso) # ---------------------------------------------------------------------- # Locate valid pixels for PCA # ---------------------------------------------------------------------- # determining the pixels relevant for PCA construction keep = np.isfinite(np.sum(abso, axis=0)) # log fraction of valid (non NaN) pixels fraction = np.nansum(keep) / len(keep) wmsg = 'Fraction of valid pixels (not NaNs) for PCA construction = {0:.3f}' WLOG(p, '', wmsg.format(fraction)) # log fraction of valid pixels > 1 - (1/e) with warnings.catch_warnings(record=True) as w: keep &= np.min(log_abso, axis=0) > -1 fraction = np.nansum(keep) / len(keep) wmsg = 'Fraction of valid pixels with transmission > 1 - (1/e) = {0:.3f}' WLOG(p, '', wmsg.format(fraction)) # ---------------------------------------------------------------------- # Perform PCA analysis on the log of the telluric absorption map # ---------------------------------------------------------------------- # Requires p: # TELLU_NUMBER_OF_PRINCIPLE_COMP # ADD_DERIV_PC # FIT_DERIV_PC # Requires loc: # DATA # Returns loc: # PC # NPC # FIT_PC loc = spirouTelluric.CalculateAbsorptionPCA(p, loc, log_abso, keep) # Plot PCA components # debug plot if p['DRS_PLOT'] and (p['DRS_DEBUG'] > 1): # plot the transmission map plot sPlt.tellu_pca_comp_plot(p, loc) # ---------------------------------------------------------------------- # Get master wavelength grid for shifting # ---------------------------------------------------------------------- # get master wave map loc['MASTERWAVEFILE'] = spirouDB.GetDatabaseMasterWave(p) loc.set_source('MASTERWAVEFILE', main_name) # log progress wmsg1 = 'Getting master wavelength grid' wmsg2 = '\tFile = {0}'.format(os.path.basename(loc['MASTERWAVEFILE'])) WLOG(p, '', [wmsg1, wmsg2]) # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # read master wave map mout = spirouImage.GetWaveSolution(p, filename=loc['MASTERWAVEFILE'], return_wavemap=True, quiet=True, fiber=wave_fiber) _, loc['MASTERWAVE'], _ = mout loc.set_source('MASTERWAVE', main_name) # ---------------------------------------------------------------------- # Loop around telluric files # ---------------------------------------------------------------------- for basefilename in p['ARG_FILE_NAMES']: # ------------------------------------------------------------------ # Construct absolute file path # ------------------------------------------------------------------ filename = os.path.join(p['ARG_FILE_DIR'], basefilename) # ------------------------------------------------------------------ # Construct output file names # ------------------------------------------------------------------ outfile1, tag1 = CONSTANTS.TELLU_FIT_OUT_FILE(p, filename) outfilename1 = os.path.basename(outfile1) outfile2, tag2 = CONSTANTS.TELLU_FIT_RECON_FILE(p, filename) outfilename2 = os.path.basename(outfile2) # ------------------------------------------------------------------ # Read filename # ------------------------------------------------------------------ # read image tdata, thdr, _, _ = spirouImage.ReadImage(p, filename) # normalise with blaze function loc['SP'] = tdata / loc['NBLAZE'] loc.set_source('SP', main_name) # ------------------------------------------------------------------ # check that file has valid DPRTYPE # ------------------------------------------------------------------ # get FP_FP DPRTYPE p = spirouImage.ReadParam(p, thdr, 'KW_DPRTYPE', 'DPRTYPE', dtype=str) # if dprtype is incorrect skip if p['DPRTYPE'] not in p['ALLOWED_TELLURIC_DPRTYPES']: wmsg1 = 'Skipping file (DPRTYPE incorrect)' wmsg2 = '\t DPRTYPE = {0}'.format(p['DPRTYPE']) WLOG(p, 'warning', [wmsg1, wmsg2]) continue # ------------------------------------------------------------------ # Set storage # ------------------------------------------------------------------ loc['RECON_ABSO'] = np.ones(np.product(loc['DATA'].shape)) loc['AMPS_ABSOL_TOTAL'] = np.zeros(loc['NPC']) loc.set_sources(['RECON_ABSO', 'AMPS_ABSOL_TOTAL'], main_name) # ------------------------------------------------------------------ # Read wavelength solution # ------------------------------------------------------------------ # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # get wavelength solution wout = spirouImage.GetWaveSolution(p, image=tdata, hdr=thdr, return_wavemap=True, return_filename=True, fiber=wave_fiber) _, loc['WAVE_IT'], loc['WAVEFILE'], loc['WSOURCE'] = wout loc.set_sources(['WAVE_IT', 'WAVEFILE', 'WSOURCE'], main_name) # load wave keys loc = spirouImage.GetWaveKeys(p, loc, thdr) # ------------------------------------------------------------------ # Interpolate at shifted wavelengths (if we have a template) # ------------------------------------------------------------------ if loc['FLAG_TEMPLATE']: # Requires p: # TELLU_FIT_KEEP_FRAC # Requires loc: # DATA # TEMPLATE # WAVE_IT # Returns: # TEMPLATE2 loc = spirouTelluric.BervCorrectTemplate(p, loc, thdr) # debug plot if p['DRS_PLOT'] and (p['DRS_DEBUG'] > 1): # plot the transmission map plot sPlt.tellu_fit_tellu_spline_plot(p, loc) # store PC and TAPAS_ALL_SPECIES before shift loc['PC_PRESHIFT'] = np.array(loc['PC']) loc['TAPAS_ALL_PRESHIFT'] = np.array(loc['TAPAS_ALL_SPECIES']) loc.set_sources(['PC_PRESHIFT', 'TAPAS_ALL_PRESHIFT'], main_name) # ------------------------------------------------------------------ # Shift the pca components to correct frame # ------------------------------------------------------------------ # log process wmsg1 = 'Shifting PCA components from master wavelength grid' wmsg2 = '\tFile = {0}'.format(os.path.basename(loc['MASTERWAVEFILE'])) WLOG(p, '', [wmsg1, wmsg2]) # shift pca components (one by one) for comp in range(loc['NPC']): wargs = [p, loc['PC'][:, comp], loc['MASTERWAVE'], loc['WAVE_IT']] shift_pc = spirouTelluric.Wave2Wave(*wargs, reshape=True) loc['PC'][:, comp] = shift_pc.reshape(wargs[1].shape) wargs = [ p, loc['FIT_PC'][:, comp], loc['MASTERWAVE'], loc['WAVE_IT'] ] shift_fpc = spirouTelluric.Wave2Wave(*wargs, reshape=True) loc['FIT_PC'][:, comp] = shift_fpc.reshape(wargs[1].shape) # ------------------------------------------------------------------ # Shift the tapas spectrum to correct frame # ------------------------------------------------------------------ # log process wmsg1 = 'Shifting TAPAS spectrum from master wavelength grid' wmsg2 = '\tFile = {0}'.format(os.path.basename(loc['MASTERWAVEFILE'])) WLOG(p, '', [wmsg1, wmsg2]) # shift tapas for comp in range(len(loc['TAPAS_ALL_SPECIES'])): wargs = [ p, loc['TAPAS_ALL_SPECIES'][comp], loc['MASTERWAVE'], loc['WAVE_IT'] ] stapas = spirouTelluric.Wave2Wave(*wargs, reshape=True) loc['TAPAS_ALL_SPECIES'][comp] = stapas.reshape(wargs[1].shape) # Debug plot to test shifting if p['DRS_PLOT'] and p['DRS_DEBUG'] > 1: sPlt.tellu_fit_debug_shift_plot(p, loc) # ------------------------------------------------------------------ # Calculate reconstructed absorption # ------------------------------------------------------------------ # Requires p: # TELLU_FIT_MIN_TRANSMISSION # TELLU_FIT_NITER # TELLU_LAMBDA_MIN # TELLU_LAMBDA_MAX # TELLU_FIT_VSINI # TRANSMISSION_CUT # FIT_DERIV_PC # LOG_OPT # Requires loc: # FLAG_TEMPLATE # TAPAS_ALL_SPECIES # AMPS_ABSOL_TOTAL # WAVE_IT # TEMPLATE2 # FIT_PC # NPC # PC # Returns loc: # SP2 # TEMPLATE2 # RECON_ABSO # AMPS_ABSOL_TOTAL loc = spirouTelluric.CalcReconAbso(p, loc) # debug plot if p['DRS_PLOT'] > 0: # plot the recon abso plot sPlt.tellu_fit_recon_abso_plot(p, loc) # ------------------------------------------------------------------ # Get molecular absorption # ------------------------------------------------------------------ # Requires p: # TELLU_FIT_LOG_LIMIT # Requeres loc: # RECON_ABSO # TAPAS_ALL_SPECIES # Returns loc: # TAPAS_{molecule} loc = spirouTelluric.CalcMolecularAbsorption(p, loc) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # ---------------------------------------------------------------------- # get SNR for each order from header nbo = loc['DATA'].shape[0] snr_order = p['QC_FIT_TELLU_SNR_ORDER'] snr = spirouImage.Read1Dkey(p, thdr, p['kw_E2DS_SNR'][0], nbo) # check that SNR is high enough if snr[snr_order] < p['QC_FIT_TELLU_SNR_MIN']: fmsg = 'low SNR in order {0}: ({1:.2f} < {2:.2f})' fargs = [snr_order, snr[snr_order], p['QC_FIT_TELLU_SNR_MIN']] fail_msg.append(fmsg.format(*fargs)) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(snr[snr_order]) qc_name_str = 'SNR[{0}]'.format(snr_order) qc_names.append(qc_name_str) qc_logic.append('{0} < {1:.2f}'.format(qc_name_str, p['QC_FIT_TELLU_SNR_ORDER'])) # ---------------------------------------------------------------------- # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') continue # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # Get components amplitudes for header # ------------------------------------------------------------------ # get raw file name raw_in_file = os.path.basename(p['FITSFILENAME']) # copy original keys hdict = spirouImage.CopyOriginalKeys(thdr) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) # set the input files hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'], value=p['BLAZFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBWAVE'], value=loc['WAVEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVESOURCE'], value=loc['WSOURCE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=p['ARG_FILE_NAMES']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # set tellu keys npc = loc['NPC'] hdict = spirouImage.AddKey(p, hdict, p['KW_TELLU_NPC'], value=npc) hdict = spirouImage.AddKey(p, hdict, p['KW_TELLU_FIT_DPC'], value=p['FIT_DERIV_PC']) hdict = spirouImage.AddKey(p, hdict, p['KW_TELLU_ADD_DPC'], value=p['ADD_DERIV_PC']) if p['ADD_DERIV_PC']: values = loc['AMPS_ABSOL_TOTAL'][:npc - 2] hdict = spirouImage.AddKey1DList(p, hdict, p['KW_TELLU_AMP_PC'], values=values, dim1name='amp') hdict = spirouImage.AddKey(p, hdict, p['KW_TELLU_DV_TELL1'], value=loc['AMPS_ABSOL_TOTAL'][npc - 2]) hdict = spirouImage.AddKey(p, hdict, p['KW_TELLU_DV_TELL2'], value=loc['AMPS_ABSOL_TOTAL'][npc - 1]) else: values = loc['AMPS_ABSOL_TOTAL'][:npc] hdict = spirouImage.AddKey1DList(p, hdict, p['KW_TELLU_AMP_PC'], values=values, dim1name='PC') # ------------------------------------------------------------------ # Write corrected spectrum to E2DS # ------------------------------------------------------------------ # reform the E2DS sp_out = loc['SP2'] / loc['RECON_ABSO'] sp_out = sp_out.reshape(loc['DATA'].shape) # multiply by blaze sp_out = sp_out * loc['NBLAZE'] hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1) # log progress wmsg = 'Saving {0} to file'.format(outfilename1) WLOG(p, '', wmsg) # write sp_out to file p = spirouImage.WriteImage(p, outfile1, sp_out, hdict) # ------------------------------------------------------------------ # 1-dimension spectral S1D (uniform in wavelength) # ------------------------------------------------------------------ # get arguments for E2DS to S1D e2dsargs = [loc['WAVE'], sp_out, loc['BLAZE']] # get 1D spectrum xs1d1, ys1d1 = spirouImage.E2DStoS1D(p, *e2dsargs, wgrid='wave') # Plot the 1D spectrum if p['DRS_PLOT'] > 0: sPlt.ext_1d_spectrum_plot(p, xs1d1, ys1d1) # construct file name targs = [p, raw_in_file] s1dfile1, tag3 = spirouConfig.Constants.TELLU_FIT_S1D_FILE1(*targs) s1dfilename1 = os.path.basename(s1dfile1) # add header keys # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3) hdict = spirouImage.AddKey(p, hdict, p['KW_EXT_TYPE'], value=p['DPRTYPE']) # log writing to file wmsg = 'Saving 1D spectrum (uniform in wavelength) in {0}' WLOG(p, '', wmsg.format(s1dfilename1)) # Write to file columns = ['wavelength', 'flux', 'eflux'] values = [xs1d1, ys1d1, np.zeros_like(ys1d1)] units = ['nm', None, None] s1d1 = spirouImage.MakeTable(p, columns, values, units=units) spirouImage.WriteTable(p, s1d1, s1dfile1, header=hdict) # ------------------------------------------------------------------ # 1-dimension spectral S1D (uniform in velocity) # ------------------------------------------------------------------ # get arguments for E2DS to S1D e2dsargs = [loc['WAVE'], sp_out, loc['BLAZE']] # get 1D spectrum xs1d2, ys1d2 = spirouImage.E2DStoS1D(p, *e2dsargs, wgrid='velocity') # Plot the 1D spectrum if p['DRS_PLOT'] > 0: sPlt.ext_1d_spectrum_plot(p, xs1d2, ys1d2) # construct file name targs = [p, raw_in_file] s1dfile2, tag4 = spirouConfig.Constants.TELLU_FIT_S1D_FILE2(*targs) s1dfilename2 = os.path.basename(s1dfile2) # add header keys hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4) hdict = spirouImage.AddKey(p, hdict, p['KW_EXT_TYPE'], value=p['DPRTYPE']) # log writing to file wmsg = 'Saving 1D spectrum (uniform in velocity) in {0}' WLOG(p, '', wmsg.format(s1dfilename2)) # Write to file columns = ['wavelength', 'flux', 'eflux'] values = [xs1d2, ys1d2, np.zeros_like(ys1d2)] units = ['nm', None, None] s1d2 = spirouImage.MakeTable(p, columns, values, units=units) spirouImage.WriteTable(p, s1d2, s1dfile2, header=hdict) # ------------------------------------------------------------------ # Write reconstructed absorption to E2DS # ------------------------------------------------------------------ # set up empty storage recon_abso2 = np.zeros_like(loc['DATA']) # get dimensions of data ydim, xdim = loc['DATA'].shape # loop around orders for order_num in range(ydim): # get start and end points start, end = xdim * order_num, xdim * order_num + xdim # save to storage recon_abso2[order_num, :] = loc['RECON_ABSO'][start:end] # add molecular absorption to file for it, molecule in enumerate(p['TELLU_ABSORBERS'][1:]): # get molecule keyword store and key molkey = '{0}_{1}'.format(p['KW_TELLU_ABSO'][0], molecule.upper()) molkws = [molkey, 0, 'Absorption in {0}'.format(molecule.upper())] # load into hdict hdict = spirouImage.AddKey(p, hdict, molkws, value=loc[molkey]) # add water col if molecule == 'h2o': loc['WATERCOL'] = loc[molkey] # set source loc.set_source('WATERCOL', main_name) # add the tau keys hdict = spirouImage.AddKey(p, hdict, p['KW_TAU_H2O'], value=loc['TAU_H2O']) hdict = spirouImage.AddKey(p, hdict, p['KW_TAU_REST'], value=loc['TAU_REST']) # log progress wmsg = 'Saving {0} to file'.format(outfilename2) WLOG(p, '', wmsg) # write recon_abso to file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2) p = spirouImage.WriteImage(p, outfile2, recon_abso2, hdict) # ------------------------------------------------------------------ # Update the Telluric database # ------------------------------------------------------------------ if p['QC']: # add TELLU_OBJ to telluric database oparams = dict(objname=loc['OBJNAME'], berv=loc['BERV'], airmass=loc['AIRMASS'], watercol=loc['WATERCOL']) spirouDB.UpdateDatabaseTellObj(p, outfilename1, **oparams) # copy file to database spirouDB.PutTelluFile(p, outfile1) # add TELLU_RECON to telluric database # add TELLU_OBJ to telluric database oparams = dict(objname=loc['OBJNAME'], berv=loc['BERV'], airmass=loc['AIRMASS'], watercol=loc['WATERCOL']) spirouDB.UpdateDatabaseTellRecon(p, outfilename2, **oparams) # copy file to database spirouDB.PutTelluFile(p, outfile2) # ---------------------------------------------------------------------- # End plotting # ---------------------------------------------------------------------- # debug plot if p['DRS_PLOT'] > 0: # end interactive session sPlt.end_interactive_session(p) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
>>> r = sdf.SDFReader() >>> m = r.read('model://pr2/model.sdf') >>> w = VRMLWriter() >>> w.write(m, '/tmp/pr2.wrl') """ from . import model from . import utils import os import sys import time import subprocess import atexit import logging import warnings with warnings.catch_warnings(): warnings.simplefilter('ignore') from .thirdparty import transformations as tf import math import numpy import copy import jinja2 import uuid try: import CORBA import CosNaming import OpenHRP except ImportError: print "Unable to find CORBA and OpenHRP library." print "You can install the library by:" print "$ sudo add-apt-repository ppa:hrg/daily"
return out try: np.array(5).astype(float, copy=False) except TypeError: # Compat where astype accepted no copy argument def astype(array, dtype, copy=True): if not copy and array.dtype == dtype: return array return array.astype(dtype) else: astype = np.ndarray.astype try: with warnings.catch_warnings(record=True): # Don't raise the numpy deprecation warnings that appear in # 1.9, but avoid Python bug due to simplefilter('ignore') warnings.simplefilter('always') sp.csr_matrix([1.0, 2.0, 3.0]).max(axis=0) except (TypeError, AttributeError): # in scipy < 14.0, sparse matrix min/max doesn't accept an `axis` argument # the following code is taken from the scipy 0.14 codebase def _minor_reduce(X, ufunc): major_index = np.flatnonzero(np.diff(X.indptr)) if X.data.size == 0 and major_index.size == 0: # Numpy < 1.8.0 don't handle empty arrays in reduceat value = np.zeros_like(X.data) else: value = ufunc.reduceat(X.data, X.indptr[major_index])
def main(argv): if not check_output('which ceph')[0]: logger.error("No 'ceph' command available. Run this script from node, which has ceph access") return # TODO: Logs from down OSD opts = parse_args(argv) res_q = Queue.Queue() run_q = Queue.Queue() with warnings.catch_warnings(): warnings.simplefilter("ignore") out_folder = os.tempnam() os.makedirs(out_folder) setup_loggers(getattr(logging, opts.log_level), os.path.join(out_folder, "log.txt")) global logger_ready logger_ready = True global SSH_OPTS SSH_OPTS = SSH_OPTS.format(opts.ssh_conn_timeout, opts.ssh_private_key) global USERNAME USERNAME = opts.username collector_settings = CollectSettings() map(collector_settings.disable, opts.disable) allowed_collectors = opts.collectors.split(',') collectors = [] if CephDataCollector.name in allowed_collectors: ceph_collector = CephDataCollector(opts, collector_settings, res_q) collectors.append(ceph_collector) else: ceph_collector = None if NodeCollector.name in allowed_collectors: node_collector = NodeCollector(opts, collector_settings, res_q) collectors.append(node_collector) else: node_collector = None if NodeResourseUsageCollector.name in allowed_collectors: node_resource_collector = NodeResourseUsageCollector(opts, collector_settings, res_q) else: node_resource_collector = None if CephPerformanceCollector.name in allowed_collectors: if CephDataCollector.name not in allowed_collectors: logger.error("Can't collect performance info without ceph info collected") exit(1) else: ceph_performance_collector = CephPerformanceCollector(opts, collector_settings, res_q) else: ceph_performance_collector = None nodes = discover_nodes(opts) nodes['master'][None] = [{}] for role, nodes_with_args in nodes.items(): if role == 'node': continue logger.info("Found %s hosts with role %s", len(nodes_with_args), role) logger.info("Found %s services with role %s", sum(map(len, nodes_with_args.values())), role) logger.info("Found %s hosts total", len(nodes['node'])) good_hosts = set(get_sshable_hosts(nodes['node'].keys())) bad_hosts = set(nodes['node'].keys()) - good_hosts if len(bad_hosts) != 0: logger.warning("Next hosts aren't awailable over ssh and would be skipped: %s", ",".join(bad_hosts)) res_q.put((True, "bad_hosts", 'json', json.dumps(list(bad_hosts)))) new_nodes = collections.defaultdict(lambda: {}) for role, role_objs in nodes.items(): if role == 'master': new_nodes[role] = role_objs else: for node, args in role_objs.items(): if node in good_hosts: new_nodes[role][node] = args nodes = new_nodes # collect data at the beginning if node_resource_collector is not None: for node, _ in nodes['node'].items(): run_q.put((node_resource_collector.collect_node, "", node, {})) for role, nodes_with_args in nodes.items(): for collector in collectors: if hasattr(collector, 'collect_' + role): coll_func = getattr(collector, 'collect_' + role) for node, kwargs_list in nodes_with_args.items(): for kwargs in kwargs_list: run_q.put((coll_func, "", node, kwargs)) save_results_thread = threading.Thread(target=save_results_th_func, args=(opts, res_q, out_folder)) save_results_thread.daemon = True save_results_thread.start() t1 = time.time() try: run_all(opts, run_q) # collect data at the end if node_resource_collector is not None: dt = opts.usage_collect_interval - (time.time() - t1) if dt > 0: logger.info("Will wait for {0} seconds for usage data collection".format(int(dt))) for i in range(int(dt / 0.1)): time.sleep(0.1) logger.info("Start final usage collection") for node, _ in nodes['node'].items(): run_q.put((node_resource_collector.collect_node, "", node, {})) run_all(opts, run_q) if ceph_performance_collector is not None: logger.info("Start performace monitoring.") with ceph_collector.osd_devs_lock: osd_devs = ceph_collector.osd_devs.copy() per_node = collections.defaultdict(lambda: []) for node, data_dev, j_dev in osd_devs.values(): per_node[node].extend((data_dev, j_dev)) # start monitoring for node, data in per_node.items(): run_q.put((ceph_performance_collector.start_performance_monitoring, "", node, {'osd_devs': data})) run_all(opts, run_q) dt = opts.performance_collect_seconds logger.info("Will wait for {0} seconds for performance data collection".format(int(dt))) for i in range(int(dt / 0.1)): time.sleep(0.1) # collect results for node, data in per_node.items(): run_q.put((ceph_performance_collector.collect_performance_data, "", node, {})) run_all(opts, run_q) except Exception: logger.exception("When collecting data:") finally: res_q.put(None) # wait till all data collected save_results_thread.join() if opts.result is None: with warnings.catch_warnings(): warnings.simplefilter("ignore") out_file = os.tempnam() + ".tar.gz" else: out_file = opts.result check_output("cd {0} ; tar -zcvf {1} *".format(out_folder, out_file)) logger.info("Result saved into %r", out_file) if opts.log_level in ('WARNING', 'ERROR', "CRITICAL"): print "Result saved into %r" % (out_file,) if not opts.dont_remove_unpacked: shutil.rmtree(out_folder) else: logger.info("Temporary folder %r", out_folder) if opts.log_level in ('WARNING', 'ERROR', "CRITICAL"): print "Temporary folder %r" % (out_folder,)
def rsa_encrypt(key, plaintext): pubkey = RSA.importKey(key) cipher = PKCS1_v1_5.new(pubkey) with warnings.catch_warnings(): warnings.simplefilter("ignore") return base64.b64encode(cipher.encrypt(plaintext))
def inner(*args, **kwargs): with warnings.catch_warnings(record=True): warnings.simplefilter("ignore") response = f(*args, **kwargs) return response
def __init__(self, filename): """Create an Overview object using an "*overview.dat" filename Parameters ---------- filename : str - should be a valid "overview.dat" style filename (see Output/ascii.cxx) Notes ----- We can currently parse the following attributes: Metallicity : float [ mass fraction ] Background Density : float [ g cm^-3 ] Background Temperature : float [ K ] With Cooling : bool Cooling Type : str Number of SNe : int Cluster Mass : float [g] seed : int Mass Loss : str """ super(Overview, self).__init__() if "_" in filename: self.id = os.path.basename(filename).split("_")[0] else: self.id = "" self.dirname = os.path.dirname(filename) # Add trailing slash (if dirname isn't empty) self.dirname = os.path.join(self.dirname, "") inputs_filename = os.path.join( os.path.dirname(filename), os.path.basename(filename).replace("overview", "inputs")) self.inputs = Inputs(inputs_filename) # default, since earlier runs won't have this saved self.cluster_mass = 0 self.cooling_type = "equilibrium" self.mass_loss = "none" f = open(filename, "r") for line in f: if "Metallicity" in line: self.metallicity = float(line.split()[1]) elif "Background Density" in line: self.background_density = float(line.split()[2]) elif "Background Temperature" in line: self.background_temperature = float(line.split()[2]) elif "With cooling" in line: self.with_cooling = string_to_bool(line.split()[2]) elif "Cooling Type" in line: self.cooling_type = line.split()[-1] elif "Number of SNe" in line: self.num_SNe = int(line.split()[-1]) elif "Cluster Mass" in line: self.cluster_mass = float(line.split()[-1]) * M_solar elif "seed" in line: self.seed = int(line.split()[-1]) elif "mass loss" in line: self.mass_loss = line.split()[-1] f.close() SNe_filename = os.path.join( os.path.dirname(filename), os.path.basename(filename).replace("overview", "SNe")) with warnings.catch_warnings(): warnings.filterwarnings("ignore", message='loadtxt: Empty input file: "*"') SNe = np.loadtxt(SNe_filename, ndmin=2, usecols=range(5)) if (SNe.shape[0] != self.num_SNe): raise ValueError("Number of SNe in datafile " + "doesn't match number listed in overview file" + " for file: " + filename) if self.num_SNe > 0: self.SNe_times = SNe[:, 0] self.SNe_initial_mass = SNe[:, 1] self.SNe_ejecta_mass = SNe[:, 2] self.SNe_ejecta_mass_Z = SNe[:, 3] self.SNe_wind_mass = SNe[:, 4] sorted_indices = np.argsort(self.SNe_times) self.SNe_times = self.SNe_times[sorted_indices] self.SNe_initial_mass = self.SNe_initial_mass[sorted_indices] self.SNe_ejecta_mass = self.SNe_ejecta_mass[sorted_indices] self.SNe_ejecta_mass_Z = self.SNe_ejecta_mass_Z[sorted_indices] self.SNe_wind_mass = self.SNe_wind_mass[sorted_indices] else: self.SNe_times = np.array([]) self.SNe_initial_mass = np.array([]) self.SNe_ejecta_mass = np.array([]) self.SNe_ejecta_mass_Z = np.array([]) self.SNe_wind_mass = np.array([]) return
def test_dimension_normalization(): with warnings.catch_warnings(record=True) as w: assert Ray((1, 1), (2, 1, 2)) == Ray((1, 1, 0), (2, 1, 2)) assert len(w) == 1
def test_iloc_getitem_frame(self): df = DataFrame(np.random.randn(10, 4), index=lrange(0, 20, 2), columns=lrange(0, 8, 2)) result = df.iloc[2] with catch_warnings(record=True): filterwarnings("ignore", "\\n.ix", DeprecationWarning) exp = df.ix[4] tm.assert_series_equal(result, exp) result = df.iloc[2, 2] with catch_warnings(record=True): filterwarnings("ignore", "\\n.ix", DeprecationWarning) exp = df.ix[4, 4] assert result == exp # slice result = df.iloc[4:8] with catch_warnings(record=True): filterwarnings("ignore", "\\n.ix", DeprecationWarning) expected = df.ix[8:14] tm.assert_frame_equal(result, expected) result = df.iloc[:, 2:3] with catch_warnings(record=True): filterwarnings("ignore", "\\n.ix", DeprecationWarning) expected = df.ix[:, 4:5] tm.assert_frame_equal(result, expected) # list of integers result = df.iloc[[0, 1, 3]] with catch_warnings(record=True): filterwarnings("ignore", "\\n.ix", DeprecationWarning) expected = df.ix[[0, 2, 6]] tm.assert_frame_equal(result, expected) result = df.iloc[[0, 1, 3], [0, 1]] with catch_warnings(record=True): filterwarnings("ignore", "\\n.ix", DeprecationWarning) expected = df.ix[[0, 2, 6], [0, 2]] tm.assert_frame_equal(result, expected) # neg indices result = df.iloc[[-1, 1, 3], [-1, 1]] with catch_warnings(record=True): filterwarnings("ignore", "\\n.ix", DeprecationWarning) expected = df.ix[[18, 2, 6], [6, 2]] tm.assert_frame_equal(result, expected) # dups indices result = df.iloc[[-1, -1, 1, 3], [-1, 1]] with catch_warnings(record=True): filterwarnings("ignore", "\\n.ix", DeprecationWarning) expected = df.ix[[18, 18, 2, 6], [6, 2]] tm.assert_frame_equal(result, expected) # with index-like s = Series(index=lrange(1, 5)) result = df.iloc[s.index] with catch_warnings(record=True): filterwarnings("ignore", "\\n.ix", DeprecationWarning) expected = df.ix[[2, 4, 6, 8]] tm.assert_frame_equal(result, expected)
def keep_stable_genes(expression, threshold=0.9, percentile=True, rank=True, return_stability=False): """ Removes genes in `expression` with differential stability < `threshold` Calculates the similarity of gene expression across brain regions for every pair of donors in `expression`. Similarity is averaged across donor pairs and genes whose mean similarity falls below `threshold` are removed. Parameters ---------- expression : list of (R, G) pandas.DataFrame Where each entry is the microarray expression of `R` regions across `G` genes for a given donor threshold : [0, 1] float, optional Minimum required average similarity (e.g, correlation) across donors for a gene to be retained. Default: 0.1 percentile : bool, optional Whether to treat `threshold` as a percentile instead of an absolute cutoff. For example, `threshold=0.9` and `percentile=True` would retain only those genes with a differential stability in the top 10% of all genes, whereas `percentile=False` would retain only those genes with differential stability > 0.9. Default: True rank : bool, optional Whether to calculate similarity as Spearman correlation instead of Pearson correlation. Default: True return_stability : bool, optional Whether to return stability estimates for each gene in addition to expression data. Default: False Returns ------- expression : list of (R, Gr) pandas.DataFrame Microarray expression for `R` regions across `Gr` genes, where `Gr` is the number of retained genes stability : (G,) numpy.ndarray Stability (average correlation) of each gene across pairs of donors. Only returned if ``return_stability=True`` """ # get number of donors and number of genes num_subj = len(expression) num_gene = expression[0].shape[-1] # rank data, if necessary for_corr = expression if not rank else [e.rank() for e in expression] # get correlation of gene expression across regions for all donor pairs gene_corrs = np.zeros((num_gene, sum(range(num_subj)))) for n, (s1, s2) in enumerate(itertools.combinations(range(num_subj), 2)): regions = np.intersect1d(for_corr[s1].dropna(axis=0, how='all').index, for_corr[s2].dropna(axis=0, how='all').index) gene_corrs[:, n] = utils.efficient_corr(for_corr[s1].loc[regions], for_corr[s2].loc[regions]) # average similarity across donors (ignore NaNs) with warnings.catch_warnings(): warnings.filterwarnings('ignore', category=RuntimeWarning, message='Mean of empty slice') gene_corrs = np.nan_to_num(np.nanmean(gene_corrs, axis=1)) # calculate absolute threshold if percentile is desired if percentile: threshold = np.percentile(gene_corrs, threshold * 100) keep_genes = gene_corrs >= threshold expression = [e.iloc[:, keep_genes] for e in expression] if return_stability: return expression, gene_corrs return expression
def test_iloc_mask(self): # GH 3631, iloc with a mask (of a series) should raise df = DataFrame(lrange(5), list('ABCDE'), columns=['a']) mask = (df.a % 2 == 0) msg = ("iLocation based boolean indexing cannot use an indexable as" " a mask") with pytest.raises(ValueError, match=msg): df.iloc[mask] mask.index = lrange(len(mask)) msg = ("iLocation based boolean indexing on an integer type is not" " available") with pytest.raises(NotImplementedError, match=msg): df.iloc[mask] # ndarray ok result = df.iloc[np.array([True] * len(mask), dtype=bool)] tm.assert_frame_equal(result, df) # the possibilities locs = np.arange(4) nums = 2 ** locs reps = lmap(bin, nums) df = DataFrame({'locs': locs, 'nums': nums}, reps) expected = { (None, ''): '0b1100', (None, '.loc'): '0b1100', (None, '.iloc'): '0b1100', ('index', ''): '0b11', ('index', '.loc'): '0b11', ('index', '.iloc'): ('iLocation based boolean indexing ' 'cannot use an indexable as a mask'), ('locs', ''): 'Unalignable boolean Series provided as indexer ' '(index of the boolean Series and of the indexed ' 'object do not match', ('locs', '.loc'): 'Unalignable boolean Series provided as indexer ' '(index of the boolean Series and of the ' 'indexed object do not match', ('locs', '.iloc'): ('iLocation based boolean indexing on an ' 'integer type is not available'), } # UserWarnings from reindex of a boolean mask with catch_warnings(record=True): simplefilter("ignore", UserWarning) result = dict() for idx in [None, 'index', 'locs']: mask = (df.nums > 2).values if idx: mask = Series(mask, list(reversed(getattr(df, idx)))) for method in ['', '.loc', '.iloc']: try: if method: accessor = getattr(df, method[1:]) else: accessor = df ans = str(bin(accessor[mask]['nums'].sum())) except Exception as e: ans = str(e) key = tuple([idx, method]) r = expected.get(key) if r != ans: raise AssertionError( "[%s] does not match [%s], received [%s]" % (key, ans, r))
def on_episode_end(self, episode, logs): """ Compute and print training statistics of the episode when done """ duration = timeit.default_timer() - self.episode_start[episode] episode_steps = len(self.observations[episode]) # Format all metrics. metrics = np.array(self.metrics[episode]) metrics_template = '' metrics_variables = [] with warnings.catch_warnings(): warnings.filterwarnings('error') for idx, name in enumerate(self.metrics_names): if idx > 0: metrics_template += ', ' try: value = np.nanmean(metrics[:, idx]) metrics_template += '{}: {:f}' except Warning: value = '--' metrics_template += '{}: {}' metrics_variables += [name, value] metrics_text = metrics_template.format(*metrics_variables) nb_step_digits = str(int(np.ceil(np.log10(self.params['nb_steps']))) + 1) lam = lambda l,n : np.array([d[n] for d in l]) keys = self.observations[episode][0].keys() template = '{step: ' + nb_step_digits + 'd}/{nb_steps}: episode: {episode}, duration: {duration:.3f}s, episode steps: {episode_steps}, steps per second: {sps:.0f}, episode reward: {episode_reward:.3f}, mean reward: {reward_mean:.3f} [{reward_min:.3f}, {reward_max:.3f}], episode delay: {episode_delay:.3f}, mean delay: {delay_mean:.3f} [{delay_min:.3f}, {delay_max:.3f}], mean action: {action_mean:.3f} [{action_min:.3f}, {action_max:.3f}]' #'{step: ' + nb_step_digits + 'd}/{nb_steps}: episode: {episode}, duration: {duration:.3f}s, episode steps: {episode_steps}, steps per second: {sps:.0f}, episode reward: {episode_reward:.3f}, mean reward: {reward_mean:.3f} [{reward_min:.3f}, {reward_max:.3f}], mean action: {action_mean:.3f} [{action_min:.3f}, {action_max:.3f}], mean observation: {obs_mean:.3f} [{obs_min:.3f}, {obs_max:.3f}], {metrics}' for k in keys: template += ', mean observation '+k+': {obs_mean_'+k+':.3f} [{obs_min_'+k+':.3f}, {obs_max_'+k+':.3f}]' template += ', {metrics}' variables = { 'step': self.step, 'nb_steps': self.params['nb_steps'], 'episode': episode + 1, 'duration': duration, 'episode_steps': episode_steps, 'sps': float(episode_steps) / duration, 'episode_reward': np.sum(self.rewards[episode]), 'reward_mean': np.mean(self.rewards[episode]), 'reward_min': np.min(self.rewards[episode]), 'reward_max': np.max(self.rewards[episode]), 'episode_delay': np.sum(self.delays[episode]), 'delay_mean': np.mean(self.delays[episode]), 'delay_min': np.min(self.delays[episode]), 'delay_max': np.max(self.delays[episode]), 'action_mean': np.mean(self.actions[episode]), 'action_min': np.min(self.actions[episode]), 'action_max': np.max(self.actions[episode]), 'metrics': metrics_text } for k in keys: variables['obs_mean_'+k] = np.mean(lam(self.observations[episode], k)) variables['obs_min_'+k] = np.min(lam(self.observations[episode], k)) variables['obs_max_'+k] = np.max(lam(self.observations[episode], k)) print(template.format(**variables)) # Debug csv. with open("debug.csv", "a") as file: file.write(str("\n")+str(variables['episode_reward'])+str(",")+str(variables['reward_mean'])+str(",")+str(variables['episode_delay'])+str(",")+str(variables['delay_mean'])) file.close() # Model checkpoint. if np.sum(self.rewards[episode])>self.lastreward: previousWeights = 'checkpoint_reward_{}.h5f'.format(self.lastreward) if os.path.exists(previousWeights): os.remove(previousWeights) self.lastreward = np.sum(self.rewards[episode]) print("The reward is higher than the best one, saving checkpoint weights") newWeights = 'checkpoint_reward_{}.h5f'.format(np.sum(self.rewards[episode])) #self.model.save_weights(newWeights, overwrite=True) self.model.model.save(newWeights, overwrite=True) else: print("The reward is lower than the best one, checkpoint weights not updated") # Free up resources. del self.episode_start[episode] del self.observations[episode] del self.rewards[episode] del self.actions[episode] del self.metrics[episode] del self.delays[episode]
def setUpModule(): global WARNING_CONTEXT WARNING_CONTEXT = warnings.catch_warnings() WARNING_CONTEXT.__enter__() warnings.simplefilter('ignore', ResourceWarning)
def etree_parse(path): # type: (str) -> Any with warnings.catch_warnings(record=False): warnings.filterwarnings("ignore", category=DeprecationWarning) return ElementTree.parse(path)
def report(self, tracker_names): assert isinstance(tracker_names, (list, tuple)) # function for loading results def read_record(filename): with open(filename) as f: record = f.read().strip().split('\n') record = [[float(t) for t in line.split(',')] for line in record] return record # assume tracker_names[0] is your tracker report_dir = os.path.join(self.report_dir, tracker_names[0]) if not os.path.exists(report_dir): os.makedirs(report_dir) report_file = os.path.join(report_dir, 'performance.json') performance = {} for name in tracker_names: print('Evaluating', name) ious = {} ious_full = {} failures = {} times = {} masks = {} # frame masks for attribute tags for s, (img_files, anno, meta) in enumerate(self.dataset): seq_name = self.dataset.seq_names[s] # initialize frames scores frame_num = len(img_files) ious[seq_name] = np.full( (self.repetitions, frame_num), np.nan, dtype=float) ious_full[seq_name] = np.full( (self.repetitions, frame_num), np.nan, dtype=float) failures[seq_name] = np.full( (self.repetitions, frame_num), np.nan, dtype=float) times[seq_name] = np.full( (self.repetitions, frame_num), np.nan, dtype=float) # read results of all repetitions record_files = sorted(glob.glob(os.path.join( self.result_dir, name, 'baseline', seq_name, '%s_[0-9]*.txt' % seq_name))) boxes = [read_record(f) for f in record_files] assert all([len(b) == len(anno) for b in boxes]) # calculate frame ious with burnin bound = Image.open(img_files[0]).size seq_ious = [self._calc_iou(b, anno, bound, burnin=True) for b in boxes] ious[seq_name][:len(seq_ious), :] = seq_ious # calculate frame ious without burnin seq_ious_full = [self._calc_iou(b, anno, bound) for b in boxes] ious_full[seq_name][:len(seq_ious_full), :] = seq_ious_full # calculate frame failures seq_failures = [ [len(b) == 1 and b[0] == 2 for b in boxes_per_rep] for boxes_per_rep in boxes] failures[seq_name][:len(seq_failures), :] = seq_failures # collect frame runtimes time_file = os.path.join( self.result_dir, name, 'baseline', seq_name, '%s_time.txt' % seq_name) if os.path.exists(time_file): seq_times = np.loadtxt(time_file, delimiter=',').T times[seq_name][:len(seq_times), :] = seq_times # collect attribute masks tag_num = len(self.tags) masks[seq_name] = np.zeros((tag_num, frame_num), bool) for i, tag in enumerate(self.tags): if tag in meta: masks[seq_name][i, :] = meta[tag] # frames with no tags if 'empty' in self.tags: tag_frames = np.array([ v for k, v in meta.items() if not 'practical' in k], dtype=bool) ind = self.tags.index('empty') masks[seq_name][ind, :] = \ ~np.logical_or.reduce(tag_frames, axis=0) # concatenate frames seq_names = self.dataset.seq_names masks = np.concatenate( [masks[s] for s in seq_names], axis=1) ious = np.concatenate( [ious[s] for s in seq_names], axis=1) failures = np.concatenate( [failures[s] for s in seq_names], axis=1) with warnings.catch_warnings(): # average over repetitions warnings.simplefilter('ignore', category=RuntimeWarning) ious = np.nanmean(ious, axis=0) failures = np.nanmean(failures, axis=0) # calculate average overlaps and failures for each tag tag_ious = np.array( [np.nanmean(ious[m]) for m in masks]) tag_failures = np.array( [np.nansum(failures[m]) for m in masks]) tag_frames = masks.sum(axis=1) # remove nan values tag_ious[np.isnan(tag_ious)] = 0.0 tag_weights = tag_frames / tag_frames.sum() # calculate weighted accuracy and robustness accuracy = np.sum(tag_ious * tag_weights) robustness = np.sum(tag_failures * tag_weights) # calculate tracking speed times = np.concatenate([ t.reshape(-1) for t in times.values()]) # remove invalid values times = times[~np.isnan(times)] times = times[times > 0] if len(times) > 0: speed = np.mean(1. / times) else: speed = -1 performance.update({name: { 'accuracy': accuracy, 'robustness': robustness, 'speed_fps': speed}}) # save performance with open(report_file, 'w') as f: json.dump(performance, f, indent=4) print('Performance saved at', report_file) return performance
def test_main(): with warnings.catch_warnings(): warnings.filterwarnings("error", module="<test string>") run_unittest(GlobalTests)
def platform(aliased=0, terse=0): """ Returns a single string identifying the underlying platform with as much useful information as possible (but no more :). The output is intended to be human readable rather than machine parseable. It may look different on different platforms and this is intended. If "aliased" is true, the function will use aliases for various platforms that report system names which differ from their common names, e.g. SunOS will be reported as Solaris. The system_alias() function is used to implement this. Setting terse to true causes the function to return only the absolute minimum information needed to identify the platform. """ result = _platform_cache.get((aliased, terse), None) if result is not None: return result # Get uname information and then apply platform specific cosmetics # to it... system, node, release, version, machine, processor = uname() if machine == processor: processor = '' if aliased: system, release, version = system_alias(system, release, version) if system == 'Windows': # MS platforms rel, vers, csd, ptype = win32_ver(version) if terse: platform = _platform(system, release) else: platform = _platform(system, release, version, csd) elif system in ('Linux',): # Linux based systems with warnings.catch_warnings(): # see issue #1322 for more information warnings.filterwarnings( 'ignore', r'dist\(\) and linux_distribution\(\) ' 'functions are deprecated .*', PendingDeprecationWarning, ) distname, distversion, distid = dist('') if distname and not terse: platform = _platform(system, release, machine, processor, 'with', distname, distversion, distid) else: # If the distribution name is unknown check for libc vs. glibc libcname, libcversion = libc_ver(sys.executable) platform = _platform(system, release, machine, processor, 'with', libcname+libcversion) elif system == 'Java': # Java platforms r, v, vminfo, (os_name, os_version, os_arch) = java_ver() if terse or not os_name: platform = _platform(system, release, version) else: platform = _platform(system, release, version, 'on', os_name, os_version, os_arch) elif system == 'MacOS': # MacOS platforms if terse: platform = _platform(system, release) else: platform = _platform(system, release, machine) else: # Generic handler if terse: platform = _platform(system, release) else: bits, linkage = architecture(sys.executable) platform = _platform(system, release, machine, processor, bits, linkage) _platform_cache[(aliased, terse)] = platform return platform
def train_exact_gp(trainX, trainY, testX, testY, kind, model_kwargs, train_kwargs, devices=('cpu',), skip_posterior_variances=False, skip_random_restart=False, evaluate_on_train=True, output_device=None, record_pred_unc=False, double=False): """Create and train an exact GP with the given options""" model_kwargs = copy.copy(model_kwargs) train_kwargs = copy.copy(train_kwargs) d = trainX.shape[-1] devices = [torch.device(device) for device in devices] if output_device is None: output_device = devices[0] else: output_device = torch.device(output_device) type_ = torch.double if double else torch.float trainX = trainX.to(output_device, type_) trainY = trainY.to(output_device, type_) testX = testX.to(output_device, type_) testY = testY.to(output_device, type_) # replace with value from dataset for convenience for k, v in list(model_kwargs.items()): if isinstance(v, str) and v == 'd': model_kwargs[k] = d # Change some options just for initial training with random restarts. random_restarts = train_kwargs.pop('random_restarts', 1) init_iters = train_kwargs.pop('init_iters', 20) optimizer_ = _map_to_optim(train_kwargs.pop('optimizer')) rr_check_conv = train_kwargs.pop('rr_check_conv', False) initial_train_kwargs = copy.copy(train_kwargs) initial_train_kwargs['max_iter'] = init_iters initial_train_kwargs['check_conv'] = rr_check_conv # initial_train_kwargs['verbose'] = 0 # don't shout about it best_model, best_likelihood, best_mll = None, None, None best_loss = np.inf # TODO: move random restarts code to a train_to_convergence-like function if not skip_random_restart: # Do some number of random restarts, keeping the best one after a truncated training. for restart in range(random_restarts): # TODO: log somehow what's happening in the restarts. model, likelihood = create_exact_gp(trainX, trainY, kind, devices=devices, **model_kwargs) model = model.to(output_device, type_) # regular marginal log likelihood mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood, model) _ = train_to_convergence(model, trainX, trainY, optimizer=optimizer_, objective=mll, isloss=False, **initial_train_kwargs) model.train() output = model(trainX) loss = -mll(output, trainY).item() if loss < best_loss: best_loss = loss best_model = model best_likelihood = likelihood best_mll = mll model = best_model likelihood = best_likelihood mll = best_mll else: model, likelihood = create_exact_gp(trainX, trainY, kind, devices=devices, **model_kwargs) model = model.to(output_device, type_) mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood, model) # fit GP with warnings.catch_warnings(record=True) as w: trained_epochs = train_to_convergence(model, trainX, trainY, optimizer=optimizer_, objective=mll, isloss=False, **train_kwargs) model.eval() likelihood.eval() mll.eval() model_metrics = dict() model_metrics['trained_epochs'] = trained_epochs with torch.no_grad(): model.train() # consider prior for evaluation on train dataset likelihood.train() train_outputs = model(trainX) model_metrics['prior_train_nmll'] = -mll(train_outputs, trainY).item() with gpytorch.settings.skip_posterior_variances(skip_posterior_variances): model.eval() # Now consider posterior distributions likelihood.eval() if evaluate_on_train: train_outputs = model(trainX) model_metrics['train_mse'] = mean_squared_error(train_outputs.mean, trainY) with warnings.catch_warnings(record=True) as w2: test_outputs = model(testX) pred_mean = test_outputs.mean if not skip_posterior_variances: # model_metrics['train_nll'] = -likelihood(train_outputs).log_prob( # trainY).item() # model_metrics['test_nll'] = -likelihood(test_outputs).log_prob( # testY).item() if evaluate_on_train: model_metrics['train_nll'] = -mll(train_outputs, trainY).item() model_metrics['test_nll'] = -mll(test_outputs, testY).item() distro = likelihood(test_outputs) lower, upper = distro.confidence_region() frac = ((testY > lower) * (testY < upper)).to(torch.float).mean().item() model_metrics['test_pred_frac_in_cr'] = frac if record_pred_unc: model_metrics['test_pred_z_score'] = (testY - distro.mean) / distro.stddev # model_metrics['test_pred_var'] = distro.variance.tolist() # model_metrics['test_pred_mean'] = distro.mean.tolist() model_metrics['training_warnings'] = len(w) model_metrics['testing_warning'] = '' if len(w2) == 0 else w2[-1].message model_metrics['state_dict_file'] = _save_state_dict(model) return model_metrics, pred_mean.to('cpu', torch.float), model
def main(): """ Launches text to speech (inference). Inference is executed on a single GPU. """ torch.backends.cudnn.benchmark = True parser = argparse.ArgumentParser(description='PyTorch FastPitch Inference', allow_abbrev=False) parser = parse_args(parser) args, unk_args = parser.parse_known_args() if args.output is not None: Path(args.output).mkdir(parents=False, exist_ok=True) log_fpath = args.log_file or str(Path(args.output, 'nvlog_infer.json')) log_fpath = unique_dllogger_fpath(log_fpath) DLLogger.init(backends=[JSONStreamBackend(Verbosity.DEFAULT, log_fpath), StdOutBackend(Verbosity.VERBOSE)]) [DLLogger.log("PARAMETER", {k:v}) for k,v in vars(args).items()] device = torch.device('cuda' if args.cuda else 'cpu') if args.fastpitch is not None: generator = load_and_setup_model( 'FastPitch', parser, args.fastpitch, args.amp, device, unk_args=unk_args, forward_is_infer=True, ema=args.ema, jitable=args.torchscript) if args.torchscript: generator = torch.jit.script(generator) else: generator = None if args.waveglow is not None: with warnings.catch_warnings(): warnings.simplefilter("ignore") waveglow = load_and_setup_model( 'WaveGlow', parser, args.waveglow, args.amp, device, unk_args=unk_args, forward_is_infer=True, ema=args.ema) denoiser = Denoiser(waveglow).to(device) waveglow = getattr(waveglow, 'infer', waveglow) else: waveglow = None if len(unk_args) > 0: raise ValueError(f'Invalid options {unk_args}') fields = load_fields(args.input) batches = prepare_input_sequence( fields, device, args.batch_size, args.dataset_path, load_mels=(generator is None)) if args.include_warmup: # Use real data rather than synthetic - FastPitch predicts len for i in range(3): with torch.no_grad(): if generator is not None: b = batches[0] mel, *_ = generator(b['text'], b['text_lens']) if waveglow is not None: audios = waveglow(mel, sigma=args.sigma_infer).float() _ = denoiser(audios, strength=args.denoising_strength) gen_measures = MeasureTime() waveglow_measures = MeasureTime() gen_kw = {'pace': args.pace, 'pitch_tgt': None, 'pitch_transform': build_pitch_transformation(args)} if args.torchscript: gen_kw.pop('pitch_transform') all_utterances = 0 all_samples = 0 all_letters = 0 all_frames = 0 reps = args.repeats log_enabled = True # reps == 1 log = lambda s, d: DLLogger.log(step=s, data=d) if log_enabled else None # for repeat in (tqdm.tqdm(range(reps)) if reps > 1 else range(reps)): k=0 for rep in range(reps): for b in batches: if generator is None: log(rep, {'Synthesizing from ground truth mels'}) mel, mel_lens = b['mel'], b['mel_lens'] else: with torch.no_grad(), gen_measures: mel, mel_lens, *_ = generator( b['text'], b['text_lens'], **gen_kw) gen_infer_perf = mel.size(0) * mel.size(2) / gen_measures[-1] all_letters += b['text_lens'].sum().item() all_frames += mel.size(0) * mel.size(2) mel_fname = b['output'][k] if 'output' in b else f'audio_{k}.pt' mel_path = Path(args.output, mel_fname) torch.save(mel.cpu(), mel_path) log(rep, {"fastpitch_frames_per_sec": gen_infer_perf}) log(rep, {"fastpitch_latency": gen_measures[-1]}) k=k+1 if waveglow is not None: with torch.no_grad(), waveglow_measures: audios = waveglow(mel, sigma=args.sigma_infer) audios = denoiser(audios.float(), strength=args.denoising_strength ).squeeze(1) all_utterances += len(audios) all_samples += sum(audio.size(0) for audio in audios) waveglow_infer_perf = ( audios.size(0) * audios.size(1) / waveglow_measures[-1]) log(rep, {"waveglow_samples_per_sec": waveglow_infer_perf}) log(rep, {"waveglow_latency": waveglow_measures[-1]}) if args.output is not None and reps == 1: for i, audio in enumerate(audios): audio = audio[:mel_lens[i].item() * args.stft_hop_length] if args.fade_out: fade_len = args.fade_out * args.stft_hop_length fade_w = torch.linspace(1.0, 0.0, fade_len) audio[-fade_len:] *= fade_w.to(audio.device) audio = audio/torch.max(torch.abs(audio)) fname = b['output'][i] if 'output' in b else f'audio_{i}.wav' audio_path = Path(args.output, fname) write(audio_path, args.sampling_rate, audio.cpu().numpy()) if generator is not None and waveglow is not None: log(rep, {"latency": (gen_measures[-1] + waveglow_measures[-1])}) log_enabled = True if generator is not None: gm = np.sort(np.asarray(gen_measures)) rtf = all_samples / (all_utterances * gm.mean() * args.sampling_rate) log('avg', {"fastpitch letters/s": all_letters / gm.sum()}) log('avg', {"fastpitch_frames/s": all_frames / gm.sum()}) log('avg', {"fastpitch_latency": gm.mean()}) log('avg', {"fastpitch RTF": rtf}) log('90%', {"fastpitch_latency": gm.mean() + norm.ppf((1.0 + 0.90) / 2) * gm.std()}) log('95%', {"fastpitch_latency": gm.mean() + norm.ppf((1.0 + 0.95) / 2) * gm.std()}) log('99%', {"fastpitch_latency": gm.mean() + norm.ppf((1.0 + 0.99) / 2) * gm.std()}) if waveglow is not None: wm = np.sort(np.asarray(waveglow_measures)) rtf = all_samples / (all_utterances * wm.mean() * args.sampling_rate) log('avg', {"waveglow_samples/s": all_samples / wm.sum()}) log('avg', {"waveglow_latency": wm.mean()}) log('avg', {"waveglow RTF": rtf}) log('90%', {"waveglow_latency": wm.mean() + norm.ppf((1.0 + 0.90) / 2) * wm.std()}) log('95%', {"waveglow_latency": wm.mean() + norm.ppf((1.0 + 0.95) / 2) * wm.std()}) log('99%', {"waveglow_latency": wm.mean() + norm.ppf((1.0 + 0.99) / 2) * wm.std()}) if generator is not None and waveglow is not None: m = gm + wm rtf = all_samples / (all_utterances * m.mean() * args.sampling_rate) log('avg', {"samples/s": all_samples / m.sum()}) log('avg', {"letters/s": all_letters / m.sum()}) log('avg', {"latency": m.mean()}) log('avg', {"RTF": rtf}) log('90%', {"latency": m.mean() + norm.ppf((1.0 + 0.90) / 2) * m.std()}) log('95%', {"latency": m.mean() + norm.ppf((1.0 + 0.95) / 2) * m.std()}) log('99%', {"latency": m.mean() + norm.ppf((1.0 + 0.99) / 2) * m.std()}) DLLogger.flush()
def test_ec(ac=None, rd=None): """Test EC methods.""" if ac is None: # test type/value checking for audio_controller assert_raises(TypeError, ExperimentController, *std_args, audio_controller=1, stim_fs=44100, **std_kwargs) assert_raises(ValueError, ExperimentController, *std_args, audio_controller='foo', stim_fs=44100, **std_kwargs) assert_raises(ValueError, ExperimentController, *std_args, audio_controller=dict(TYPE='foo'), stim_fs=44100, **std_kwargs) # monitor, etc. assert_raises(TypeError, ExperimentController, *std_args, monitor='foo', **std_kwargs) assert_raises(KeyError, ExperimentController, *std_args, monitor=dict(), **std_kwargs) assert_raises(ValueError, ExperimentController, *std_args, response_device='foo', **std_kwargs) std_kwargs.update(window_size=10.) assert_raises(ValueError, ExperimentController, *std_args, **std_kwargs) std_kwargs.update(window_size=(1, 1)) assert_raises(ValueError, ExperimentController, *std_args, audio_controller='pyglet', response_device='tdt', **std_kwargs) assert_raises(ValueError, ExperimentController, *std_args, audio_controller='pyglet', response_device='keyboard', trigger_controller='tdt', **std_kwargs) # test type checking for 'session' std_kwargs['session'] = 1 assert_raises(TypeError, ExperimentController, *std_args, audio_controller='pyglet', stim_fs=44100, **std_kwargs) std_kwargs['session'] = '01' # test value checking for trigger controller assert_raises(ValueError, ExperimentController, *std_args, audio_controller='pyglet', trigger_controller='foo', stim_fs=44100, **std_kwargs) # test value checking for RMS checker assert_raises(ValueError, ExperimentController, *std_args, audio_controller='pyglet', check_rms=True, stim_fs=44100, **std_kwargs) # run rest of test with audio_controller == 'pyglet' this_ac = 'pyglet' this_rd = 'keyboard' this_tc = 'dummy' this_fs = 44100 else: assert ac == 'tdt' # run rest of test with audio_controller == 'tdt' this_ac = ac this_rd = rd this_tc = ac this_fs = get_tdt_rates()['25k'] assert_raises(ValueError, ExperimentController, *std_args, audio_controller=dict(TYPE=this_ac, TDT_MODEL='foo'), **std_kwargs) with warnings.catch_warnings(record=True) as w: for suppress in (True, False): with ExperimentController(*std_args, audio_controller=this_ac, response_device=this_rd, trigger_controller=this_tc, stim_fs=100., suppress_resamp=suppress, **std_kwargs) as ec: pass warnings.simplefilter('ignore') # ignore dummy TDT warning with ExperimentController(*std_args, audio_controller=this_ac, response_device=this_rd, trigger_controller=this_tc, stim_fs=this_fs, **std_kwargs) as ec: warnings.simplefilter('always') assert_true(ec.participant == std_kwargs['participant']) assert_true(ec.session == std_kwargs['session']) assert_true(ec.exp_name == std_args[0]) stamp = ec.current_time ec.write_data_line('hello') ec.wait_until(stamp + 0.02) ec.screen_prompt('test', 0.01, 0, None) ec.screen_prompt('test', 0.01, 0, ['1']) ec.screen_prompt(['test', 'ing'], 0.01, 0, ['1']) assert_raises(ValueError, ec.screen_prompt, 'foo', np.inf, 0, []) assert_raises(TypeError, ec.screen_prompt, 3, 0.01, 0, None) assert_equal(ec.wait_one_press(0.01), (None, None)) assert_true(ec.wait_one_press(0.01, timestamp=False) is None) assert_equal(ec.wait_for_presses(0.01), []) assert_equal(ec.wait_for_presses(0.01, timestamp=False), []) assert_raises(ValueError, ec.get_presses) ec.listen_presses() assert_equal(ec.get_presses(), []) assert_equal(ec.get_presses(kind='presses'), []) assert_raises(ValueError, ec.get_presses, kind='foo') if this_rd == 'tdt': # TDT does not have key release events, so should raise an # exception if asked for them: assert_raises(RuntimeError, ec.get_presses, kind='releases') assert_raises(RuntimeError, ec.get_presses, kind='both') else: assert_equal(ec.get_presses(kind='both'), []) assert_equal(ec.get_presses(kind='releases'), []) ec.set_noise_db(0) ec.set_stim_db(20) # test buffer data handling ec.set_rms_checking(None) ec.load_buffer([0, 0, 0, 0, 0, 0]) assert_raises(ValueError, ec.load_buffer, [0, 2, 0, 0, 0, 0]) ec.load_buffer(np.zeros((100, ))) ec.load_buffer(np.zeros((100, 1))) ec.load_buffer(np.zeros((100, 2))) ec.load_buffer(np.zeros((1, 100))) ec.load_buffer(np.zeros((2, 100))) data = np.empty(int(5e6), np.float32) # too long for TDT if this_fs == get_tdt_rates()['25k']: assert_raises(RuntimeError, ec.load_buffer, data) else: ec.load_buffer(data) ec.load_buffer(np.zeros(2)) del data assert_raises(ValueError, ec.stamp_triggers, 'foo') assert_raises(ValueError, ec.stamp_triggers, 0) assert_raises(ValueError, ec.stamp_triggers, 3) assert_raises(ValueError, ec.stamp_triggers, 1, check='foo') ec.stamp_triggers(3, check='int4') ec.stamp_triggers(2) ec.stamp_triggers([2, 4, 8]) assert_raises(ValueError, ec.load_buffer, np.zeros((100, 3))) assert_raises(ValueError, ec.load_buffer, np.zeros((3, 100))) assert_raises(ValueError, ec.load_buffer, np.zeros((1, 1, 1))) # test RMS checking assert_raises(ValueError, ec.set_rms_checking, 'foo') # click: RMS 0.0135, should pass 'fullfile' and fail 'windowed' click = np.zeros((int(ec.fs / 4), )) # 250 ms click[len(click) // 2] = 1. click[len(click) // 2 + 1] = -1. # noise: RMS 0.03, should fail both 'fullfile' and 'windowed' noise = np.random.normal(scale=0.03, size=(int(ec.fs / 4), )) ec.set_rms_checking(None) ec.load_buffer(click) # should go unchecked ec.load_buffer(noise) # should go unchecked ec.set_rms_checking('wholefile') with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') ec.load_buffer(click) # should pass assert_equal(len(w), 0) ec.load_buffer(noise) assert_equal(len(w), 1) ec.set_rms_checking('windowed') ec.load_buffer(click) assert_equal(len(w), 2) ec.load_buffer(noise) assert_equal(len(w), 3) ec.stop() ec.set_visible() ec.set_visible(False) ec.call_on_every_flip(partial(dummy_print, 'called start stimuli')) # Note: we put some wait_secs in here because otherwise the delay in # play start (e.g. for trigdel and onsetdel) can # mess things up! So we probably eventually should add # some safeguard against stopping too quickly after starting... # # First: identify_trial # noise = np.random.normal(scale=0.03, size=(int(ec.fs), )) ec.load_buffer(noise) assert_raises(RuntimeError, ec.start_stimulus) # order violation assert_true(ec._playing is False) ec.start_stimulus(start_of_trial=False) # should work ec.wait_secs(0.05) assert_true(ec._playing is True) assert_raises(RuntimeError, ec.trial_ok) # order violation ec.stop() assert_true(ec._playing is False) # only binary for TTL assert_raises(KeyError, ec.identify_trial, ec_id='foo') # need ttl_id assert_raises(TypeError, ec.identify_trial, ec_id='foo', ttl_id='bar') assert_raises(ValueError, ec.identify_trial, ec_id='foo', ttl_id=[2]) assert_true(ec._playing is False) ec.identify_trial(ec_id='foo', ttl_id=[0, 1]) assert_true(ec._playing is False) # # Second: start_stimuli # assert_raises(RuntimeError, ec.identify_trial, ec_id='foo', ttl_id=[0]) assert_true(ec._playing is False) assert_raises(RuntimeError, ec.trial_ok) # order violation assert_true(ec._playing is False) ec.start_stimulus(flip=False, when=-1) if ac != 'tdt': # dummy TDT version won't do this check properly, as # ec._ac._playing -> GetTagVal('playing') always gives False assert_raises(RuntimeError, ec.play) # already played, must stop ec.wait_secs(0.05) ec.stop() assert_true(ec._playing is False) # # Third: trial_ok # assert_raises(RuntimeError, ec.start_stimulus) # order violation assert_raises(RuntimeError, ec.identify_trial) # order violation ec.trial_ok() # double-check assert_raises(RuntimeError, ec.start_stimulus) # order violation ec.start_stimulus(start_of_trial=False) # should work assert_raises(RuntimeError, ec.trial_ok) # order violation ec.wait_secs(0.05) ec.stop() assert_true(ec._playing is False) ec.flip(-np.inf) assert_true(ec._playing is False) ec.estimate_screen_fs() assert_true(ec._playing is False) ec.play() ec.wait_secs(0.05) assert_true(ec._playing is True) ec.call_on_every_flip(None) ec.call_on_next_flip(ec.start_noise()) ec.wait_secs(0.05) ec.stop() assert_true(ec._playing is False) ec.start_stimulus(start_of_trial=False) ec.call_on_next_flip(ec.stop_noise) ec.stop() ec.start_stimulus(start_of_trial=False) ec.get_mouse_position() ec.listen_clicks() ec.get_clicks() ec.toggle_cursor(False) ec.toggle_cursor(True, True) ec.wait_secs(0.001) print(ec.id_types) print(ec.stim_db) print(ec.noise_db) print(ec.on_next_flip_functions) print(ec.on_every_flip_functions) print(ec.window) data = ec.screenshot() assert_allclose(data.shape[:2], std_kwargs['window_size']) print(ec.fs) # test fs support wait_secs(0.01) test_pix = (11.3, 0.5, 110003) print(test_pix) # test __repr__ assert all([x in repr(ec) for x in ['foo', '"test"', '01']]) del ec
def test_deprecation_warning(self): with warnings.catch_warnings(record=True) as warns: warnings.simplefilter('always') assign("contenttypes.change_contenttype", self.group) self.assertEqual(len(warns), 1) self.assertTrue(isinstance(warns[0].message, DeprecationWarning))
def test_error(self): with original_warnings.catch_warnings(module=self.module) as w: self.module.resetwarnings() self.module.filterwarnings("error", category=UserWarning) self.assertRaises(UserWarning, self.module.warn, "FilterTests.test_error")
def test_summary(self): with warnings.catch_warnings(): warnings.filterwarnings("ignore", "kurtosistest only valid for n>=20") self.model.fit().summary()
def concat_pandas( dfs, axis=0, join="outer", uniform=False, filter_warning=True, ignore_index=False ): if axis == 1: return pd.concat(dfs, axis=axis, join=join, sort=False) # Support concatenating indices along axis 0 if isinstance(dfs[0], pd.Index): if isinstance(dfs[0], pd.CategoricalIndex): for i in range(1, len(dfs)): if not isinstance(dfs[i], pd.CategoricalIndex): dfs[i] = dfs[i].astype("category") return pd.CategoricalIndex(union_categoricals(dfs), name=dfs[0].name) elif isinstance(dfs[0], pd.MultiIndex): first, rest = dfs[0], dfs[1:] if all( (isinstance(o, pd.MultiIndex) and o.nlevels >= first.nlevels) for o in rest ): arrays = [ concat([i._get_level_values(n) for i in dfs]) for n in range(first.nlevels) ] return pd.MultiIndex.from_arrays(arrays, names=first.names) to_concat = (first.values,) + tuple(k._values for k in rest) new_tuples = np.concatenate(to_concat) try: return pd.MultiIndex.from_tuples(new_tuples, names=first.names) except Exception: return pd.Index(new_tuples) return dfs[0].append(dfs[1:]) # Handle categorical index separately dfs0_index = dfs[0].index has_categoricalindex = isinstance(dfs0_index, pd.CategoricalIndex) or ( isinstance(dfs0_index, pd.MultiIndex) and any(isinstance(i, pd.CategoricalIndex) for i in dfs0_index.levels) ) if has_categoricalindex: dfs2 = [df.reset_index(drop=True) for df in dfs] ind = concat([df.index for df in dfs]) else: dfs2 = dfs ind = None # Concatenate the partitions together, handling categories as needed if ( isinstance(dfs2[0], pd.DataFrame) if uniform else any(isinstance(df, pd.DataFrame) for df in dfs2) ): if uniform: dfs3 = dfs2 cat_mask = dfs2[0].dtypes == "category" else: # When concatenating mixed dataframes and series on axis 1, Pandas # converts series to dataframes with a single column named 0, then # concatenates. dfs3 = [ df if isinstance(df, pd.DataFrame) else df.to_frame().rename(columns={df.name: 0}) for df in dfs2 ] # pandas may raise a RuntimeWarning for comparing ints and strs with warnings.catch_warnings(): warnings.simplefilter("ignore", RuntimeWarning) if filter_warning: warnings.simplefilter("ignore", FutureWarning) cat_mask = pd.concat( [(df.dtypes == "category").to_frame().T for df in dfs3], join=join, sort=False, ).any() if cat_mask.any(): not_cat = cat_mask[~cat_mask].index # this should be aligned, so no need to filter warning out = pd.concat( [df[df.columns.intersection(not_cat)] for df in dfs3], join=join, sort=False, ) temp_ind = out.index for col in cat_mask.index.difference(not_cat): # Find an example of categoricals in this column for df in dfs3: sample = df.get(col) if sample is not None: break # Extract partitions, subbing in missing if needed parts = [] for df in dfs3: if col in df.columns: parts.append(df[col]) else: codes = np.full(len(df), -1, dtype="i8") data = pd.Categorical.from_codes( codes, sample.cat.categories, sample.cat.ordered ) parts.append(data) out[col] = union_categoricals(parts) # Pandas resets index type on assignment if frame is empty # https://github.com/pandas-dev/pandas/issues/17101 if not len(temp_ind): out.index = temp_ind out = out.reindex(columns=cat_mask.index) else: # pandas may raise a RuntimeWarning for comparing ints and strs with warnings.catch_warnings(): warnings.simplefilter("ignore", RuntimeWarning) if filter_warning: warnings.simplefilter("ignore", FutureWarning) out = pd.concat(dfs3, join=join, sort=False) else: if is_categorical_dtype(dfs2[0].dtype): if ind is None: ind = concat([df.index for df in dfs2]) return pd.Series(union_categoricals(dfs2), index=ind, name=dfs2[0].name) with warnings.catch_warnings(): if filter_warning: warnings.simplefilter("ignore", FutureWarning) out = pd.concat(dfs2, join=join, sort=False) # Re-add the index if needed if ind is not None: out.index = ind return out
def read_grayscale_u8(path_to_sequence: str) -> pims.FramesSequence: with warnings.catch_warnings(): warnings.simplefilter('ignore') return _to_uint8(read_grayscale_f32(path_to_sequence))
def test_dt_accessor_api_for_categorical(self): # https://github.com/pandas-dev/pandas/issues/10661 from pandas.core.indexes.accessors import Properties s_dr = Series(date_range("1/1/2015", periods=5, tz="MET")) c_dr = s_dr.astype("category") s_pr = Series(period_range("1/1/2015", freq="D", periods=5)) c_pr = s_pr.astype("category") s_tdr = Series(timedelta_range("1 days", "10 days")) c_tdr = s_tdr.astype("category") # only testing field (like .day) # and bool (is_month_start) get_ops = lambda x: x._datetimelike_ops test_data = [ ("Datetime", get_ops(DatetimeIndex), s_dr, c_dr), ("Period", get_ops(PeriodArray), s_pr, c_pr), ("Timedelta", get_ops(TimedeltaIndex), s_tdr, c_tdr), ] assert isinstance(c_dr.dt, Properties) special_func_defs = [ ("strftime", ("%Y-%m-%d", ), {}), ("tz_convert", ("EST", ), {}), ("round", ("D", ), {}), ("floor", ("D", ), {}), ("ceil", ("D", ), {}), ("asfreq", ("D", ), {}), # FIXME: don't leave commented-out # ('tz_localize', ("UTC",), {}), ] _special_func_names = [f[0] for f in special_func_defs] # the series is already localized _ignore_names = ["tz_localize", "components"] for name, attr_names, s, c in test_data: func_names = [ f for f in dir(s.dt) if not (f.startswith("_") or f in attr_names or f in _special_func_names or f in _ignore_names) ] func_defs = [(f, (), {}) for f in func_names] for f_def in special_func_defs: if f_def[0] in dir(s.dt): func_defs.append(f_def) for func, args, kwargs in func_defs: with warnings.catch_warnings(): if func == "to_period": # dropping TZ warnings.simplefilter("ignore", UserWarning) res = getattr(c.dt, func)(*args, **kwargs) exp = getattr(s.dt, func)(*args, **kwargs) tm.assert_equal(res, exp) for attr in attr_names: res = getattr(c.dt, attr) exp = getattr(s.dt, attr) if isinstance(res, DataFrame): tm.assert_frame_equal(res, exp) elif isinstance(res, Series): tm.assert_series_equal(res, exp) else: tm.assert_almost_equal(res, exp) invalid = Series([1, 2, 3]).astype("category") msg = "Can only use .dt accessor with datetimelike" with pytest.raises(AttributeError, match=msg): invalid.dt assert not hasattr(invalid, "str")
def assert_no_user_warning(f, args): with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") f(*args) assert len(w) == 0
def test_multiple(self): # test iterating over both entries and featurizers for iter_entries in [True, False]: multi_f = MultipleFeaturizer([self.single, self.multi], iterate_over_entries=iter_entries) data = self.make_test_data() self.assertArrayAlmostEqual([2, 0, 3], multi_f.featurize(1)) self.assertArrayEqual(['A'], multi_f.citations()) implementors = multi_f.implementors() self.assertIn('Us', implementors) self.assertIn('Them', implementors) self.assertEqual(2, len(implementors)) # Ensure BaseFeaturizer operation without overriden featurize_dataframe with warnings.catch_warnings(record=True) as w: multi_f.featurize_dataframe(data, 'x') self.assertEqual(len(w), 0) self.assertArrayAlmostEqual(data['y'], [2, 3, 4]) self.assertArrayAlmostEqual(data['w'], [0, 1, 2]) self.assertArrayAlmostEqual(data['z'], [3, 4, 5]) f = MatrixFeaturizer() multi_f = MultipleFeaturizer([self.single, self.multi, f]) data = self.make_test_data() with warnings.catch_warnings(record=True) as w: multi_f.featurize_dataframe(data, 'x') self.assertEqual(len(w), 0) self.assertArrayAlmostEqual(data['representation'][0], [[1.0, 0.0], [0.0, 1.0]])
def test_tfidf_no_smoothing(): X = [[1, 1, 1], [1, 1, 0], [1, 0, 0]] tr = TfidfTransformer(smooth_idf=False, norm='l2') tfidf = tr.fit_transform(X).toarray() assert_true((tfidf >= 0).all()) # check normalization assert_array_almost_equal((tfidf ** 2).sum(axis=1), [1., 1., 1.]) # the lack of smoothing make IDF fragile in the presence of feature with # only zeros X = [[1, 1, 0], [1, 1, 0], [1, 0, 0]] tr = TfidfTransformer(smooth_idf=False, norm='l2') # First we need to verify that numpy here provides div 0 warnings with warnings.catch_warnings(record=True) as w: 1. / np.array([0.]) numpy_provides_div0_warning = len(w) == 1 with warnings.catch_warnings(record=True) as w: tfidf = tr.fit_transform(X).toarray() if not numpy_provides_div0_warning: raise SkipTest("Numpy does not provide div 0 warnings.") assert_equal(len(w), 1) # For Python 3 compatibility if hasattr(w[0].message, 'args'): assert_true("divide by zero" in w[0].message.args[0]) else: assert_true("divide by zero" in w[0].message)