def compress_cm6(data): """ CM6 compress data :type data: :class:`numpy.ndarray`, dtype=int32 :param data: the data to write :returns: NumPy chararray containing compressed samples """ data = np.ascontiguousarray(data, np.int32) n = len(data) count = [0] # closure, must be container # 4 character bytes per int32_t carr = np.zeros(n * 4, dtype=native_str('c')) def writer(char): carr[count[0]] = char count[0] += 1 return 0 cwriter = C.CFUNCTYPE(C.c_int, C.c_char)(writer) ierr = clibgse2.compress_6b_buffer(data, n, cwriter) if ierr != 0: msg = "Error status after compress_6b_buffer is NOT 0 but %d" raise GSEUtiError(msg % ierr) cnt = count[0] if cnt < 80: return carr[:cnt].view(native_str('|S%d' % cnt)) else: return carr[:(cnt // 80 + 1) * 80].view(native_str('|S80'))
def _get_platform(self): from simtk import openmm preference = ['CUDA', 'OpenCL', 'CPU', 'Reference'] from_lower = {x.lower(): x for x in preference} properties = {} if self.params.compute_platform.lower() == 'auto': for platname in preference: try: platform = openmm.Platform.getPlatformByName(platname) except Exception: # it just throws "Exception" unfortunately continue else: use_platform = platname break else: raise moldesign.NotSupportedError("Likely OpenMM installation error. " "none of the expected platforms were found: " + ', '.join(preference)) else: use_platform = self.params.compute_platform self.params.compute_platform = use_platform.lower() platform = openmm.Platform.getPlatformByName(from_lower[self.params.compute_platform]) if self.params.compute_platform == 'cpu' and self.params.num_cpus > 0: # need to use native_strs here or the swig interface gets confused properties[native_str('Threads')] = native_str(self.params.num_cpus) return platform, properties
def test_allDataTypesAndEndiansInMultipleFiles(self): """ Tests writing all different types. This is an test which is independent of the read method. Only the data part is verified. """ file = os.path.join(self.path, "data", "BW.BGLD.__.EHE.D.2008.001.first_record") # Read the data and copy them st = read(file) data_copy = st[0].data.copy() # Float64, Float32, Int32, Int24, Int16, Char encodings = {5: "f8", 4: "f4", 3: "i4", 0: "S1", 1: "i2"} byteorders = {0: '<', 1: '>'} for byteorder, btype in byteorders.items(): for encoding, dtype in encodings.items(): # Convert data to floats and write them again st[0].data = data_copy.astype(native_str(dtype)) with NamedTemporaryFile() as tf: tempfile = tf.name st.write(tempfile, format="MSEED", encoding=encoding, reclen=256, byteorder=byteorder) # Read the first record of data without header not using # ObsPy with open(tempfile, "rb") as fp: s = fp.read() data = np.fromstring(s[56:256], dtype=native_str(btype + dtype)) np.testing.assert_array_equal(data, st[0].data[:len(data)]) # Read the binary chunk of data with ObsPy st2 = read(tempfile) np.testing.assert_array_equal(st2[0].data, st[0].data)
def fetch(self, reference=None, start=None, end=None, filters=None, incl_left=True, incl_right=True): file_obj = pysam.TabixFile(native_str(self._file_path), parser=self._parser) with contextlib.closing(file_obj) as tb_file: if reference is not None: reference = native_str(reference) records = self._fetch(tb_file, reference=reference, start=start, end=end) # Filter records on additional filters. if filters is not None: def _filter(records, name, value): for r in records: if hasattr(r, name) and getattr(r, name) == value: yield r for name, value in filters.items(): records = _filter(records, name, value) # Filter inclusive/exclusive if needed. if not incl_left: records = filter(lambda r: r.start > start, records) if not incl_right: records = filter(lambda r: r.end < end, records) # Yield records. for record in records: yield record
def test_allDataTypesAndEndiansInSingleFile(self): """ Tests all data and endian types into a single file. """ with NamedTemporaryFile() as tf: tempfile = tf.name st1 = Stream() data = np.random.randint(-1000, 1000, 500) for dtype in ["i2", "i4", "f4", "f8", "S1"]: for enc in ["<", ">", "="]: typed_data = data.astype(np.dtype(native_str(enc + dtype))) st1.append(Trace(data=typed_data)) # this will raise a UserWarning - ignoring for test with warnings.catch_warnings(record=True): warnings.simplefilter('ignore', UserWarning) st1.write(tempfile, format="MSEED") # read everything back (int16 gets converted into int32) st2 = read(tempfile) for dtype in ["i4", "i4", "f4", "f8", "S1"]: for enc in ["<", ">", "="]: tr = st2.pop(0).data self.assertEqual(tr.dtype.kind + str(tr.dtype.itemsize), dtype) # byte order is always native (=) typed_data = data.astype(native_str("=" + dtype)) np.testing.assert_array_equal(tr, typed_data)
def _read_ak135_test_files(self, filename): """ Helper function parsing the AK135 test data from the original TauP test suite. """ filename = os.path.join(DATA, filename) with open(filename, "rb") as fh: line = fh.readline() line = line.strip().split() depths = list(map(float, line)) data = np.genfromtxt(fh) dist = data[:, 0] time = data[:, 1:-len(depths)] ray_param = data[:, -len(depths):] time = time[:, ::2] * 60.0 + time[:, 1::2] values = np.empty(np.size(ray_param), dtype=[(native_str('depth'), np.float_), (native_str('dist'), np.float_), (native_str('ray_param'), np.float_), (native_str('time'), np.float_)]) values['depth'] = np.tile(depths, len(dist)) values['dist'] = np.repeat(dist, len(depths)) values['ray_param'] = ray_param.flat values['time'] = time.flat return values
def __init__(self): self.quads_index = [] with open(self.names_file, 'r') as fh: self.names = [name.strip() for name in fh] with open(self.quadsindex_file, 'r') as fh: indexes = [] for index in fh: indexes += [n.strip() for n in index.split(' ') if n != ''] self.lons_per_lat = dict(zip( self.quads_order, [indexes[x:x + 91] for x in range(0, len(indexes), 91)] )) self.lat_begins = {} for quad, index in self.lons_per_lat.items(): begin = 0 end = -1 begins = [] n = 0 for item in index: n += 1 begin = end + 1 begins.append(begin) end += int(item) self.lat_begins[quad] = begins self.lons = {} self.fenums = {} for quad, sect_file in zip(self.quads_order, self.sect_files): sect = [] with open(sect_file, 'r') as fh: for line in fh: sect += [int(v) for v in line.strip().split(' ') if v != ''] lons = [] fenums = [] n = 0 for item in sect: n += 1 if n % 2: lons.append(item) else: fenums.append(item) self.lons[quad] = lons self.fenums[quad] = fenums with open(self.numbers_file, 'rt') as csvfile: fe_csv = csv.reader(csvfile, delimiter=native_str(';'), quotechar=native_str('#'), skipinitialspace=True) self.by_number = \ {int(row[0]): row[1] for row in fe_csv if len(row) > 1}
def add_unittests(testsuite, module_name): """ Function to add all available unittests of the module with given name (e.g. "obspy.core") to the given unittest TestSuite. All submodules in the "tests" directory whose names are starting with ``test_`` are added. :type testsuite: unittest.TestSuite :param testsuite: testsuite to which the tests should be added :type module_name: str :param module_name: name of the module of which the tests should be added .. rubric:: Example >>> import unittest >>> suite = unittest.TestSuite() >>> add_unittests(suite, "obspy.core") """ module_tests = __import__(module_name + ".tests", fromlist=[native_str("obspy")]) filename_pattern = os.path.join(module_tests.__path__[0], "test_*.py") files = glob.glob(filename_pattern) names = (os.path.basename(file).split(".")[0] for file in files) module_names = (".".join([module_name, "tests", name]) for name in names) for _module_name in module_names: _module = __import__(_module_name, fromlist=[native_str("obspy")]) testsuite.addTest(_module.suite())
def import_global(path): """ Import a class from a string module class path """ components = path.split(".") module = components[:-1] module = ".".join(module) mod = __import__(module, fromlist=[native_str(components[-1])]) return getattr(mod, native_str(components[-1]))
def update(self, info_hash, fields): multi_call = xmlrpc_client.MultiCall(self._server) for key, val in fields.items(): method_name = 'd.%s.set' % key getattr(multi_call, method_name)(native_str(info_hash), native_str(val)) return multi_call()[0]
def test_searchFlagInBlockette(self): """ Test case for obspy.io.mseed.util._search_flag_in_blockette """ # Write dummy file npts = 2000 np.random.seed(42) # make test reproducible data = np.random.randint(-1000, 1000, npts).astype(np.int32) # This header ensures presence of blockettes 1000 and 1001 stat_header = {'network': 'NE', 'station': 'STATI', 'location': 'LO', 'channel': 'CHA', 'npts': len(data), 'sampling_rate': 1, 'mseed': {'dataquality': 'D', 'blkt1001': {'timing_quality': 63}}} stat_header['starttime'] = UTCDateTime(datetime(2012, 8, 1, 12, 0, 0, 42)) trace1 = Trace(data=data, header=stat_header) st = Stream([trace1]) with NamedTemporaryFile() as tf: st.write(tf, format="mseed", encoding=11, reclen=512) tf.seek(0, os.SEEK_SET) file_name = tf.name with open(file_name, "rb") as file_desc: file_desc.seek(0, os.SEEK_SET) # Test from file start read_bytes = util._search_flag_in_blockette( file_desc, 48, 1001, 4, 1) self.assertFalse(read_bytes is None) self.assertEqual(unpack(native_str(">B"), read_bytes)[0], 63) # Test from middle of a record header file_desc.seek(14, os.SEEK_CUR) file_pos = file_desc.tell() read_bytes = util._search_flag_in_blockette( file_desc, 34, 1000, 6, 1) self.assertFalse(read_bytes is None) self.assertEqual(unpack(native_str(">B"), read_bytes)[0], 9) # Check that file_desc position has not changed self.assertEqual(file_desc.tell(), file_pos) # Test from middle of a record data file_desc.seek(60, os.SEEK_CUR) read_bytes = util._search_flag_in_blockette( file_desc, -26, 1001, 5, 1) self.assertFalse(read_bytes is None) self.assertEqual(unpack(native_str(">B"), read_bytes)[0], 42) # Test another record. There is at least 3 records in a # mseed with 2000 data points and 512 bytes record length file_desc.seek(1040, os.SEEK_SET) read_bytes = util._search_flag_in_blockette(file_desc, 32, 1001, 4, 1) self.assertEqual(unpack(native_str(">B"), read_bytes)[0], 63) # Test missing blockette read_bytes = util._search_flag_in_blockette(file_desc, 32, 201, 4, 4) self.assertIs(read_bytes, None)
def __new__(cls, *args, **kwargs): if (len(args) == 1 and not kwargs or not args and list(kwargs.keys()) == ['object']): return super().__new__(cls, *args, **kwargs) name = kwargs.get('name', args[0]) bases = kwargs.get('bases', args[1]) dct = kwargs.get('dict', args[2]) name = utils.native_str(name) dct = dict((utils.native_str(x), y) for x, y in dct.items()) return super().__new__(cls, name, bases, dct)
def _read_taup_output(self, filename): """ Helper method reading a stdout capture of TauP. """ with open(os.path.join(DATA, filename), "rb") as fh: while True: line = fh.readline().strip() if line.startswith(b"-----"): break output = np.genfromtxt( fh, usecols=[0, 1, 2, 3, 4, 5, 6, 7, 9], dtype=[(native_str('distance'), np.float_), (native_str('depth'), np.float_), (native_str('name'), (np.str_, 10)), (native_str('time'), np.float_), (native_str('ray_param_sec_degree'), np.float_), (native_str('takeoff_angle'), np.float_), (native_str('incident_angle'), np.float_), (native_str('purist_distance'), np.float_), (native_str('purist_name'), (np.str_, 10))]) output = np.atleast_1d(output) return output
def _create_layer(data_source, layer_name, field_definitions): sr = _get_WGS84_spatial_reference() layer = data_source.CreateLayer(native_str(layer_name), sr, ogr.wkbPoint) # Add the fields we're interested in for name, type_, width, precision in field_definitions: field = ogr.FieldDefn(native_str(name), type_) if width is not None: field.SetWidth(width) if precision is not None: field.SetPrecision(precision) layer.CreateField(field) return layer
def compare(self, reltol=1): # @UnusedVariable """ Run :func:`matplotlib.testing.compare.compare_images` and raise an unittest.TestCase.failureException with the message string given by matplotlib if the comparison exceeds the allowed tolerance. """ from matplotlib.testing.compare import compare_images if os.stat(self.name).st_size == 0: msg = "Empty output image file." raise ImageComparisonException(msg) msg = compare_images(native_str(self.baseline_image), native_str(self.name), tol=self.tol) return msg
def _print_tree(node, level=0): """ print tree with indentation """ if type(node) is list: neon_logger.display((" " * level) + ", ".join(native_str(s) for s in node[0:3])) if len(node) > 3: _print_tree(node[3], level + 1) if len(node) > 4: _print_tree(node[4], level + 1) else: neon_logger.display((" " * level) + native_str(node))
def __enter__(self): """ Set matplotlib defaults. """ MatplotlibBackend.switch_backend("AGG", sloppy=False) from matplotlib import font_manager, rcParams, rcdefaults import locale try: locale.setlocale(locale.LC_ALL, native_str('en_US.UTF-8')) except Exception: try: locale.setlocale(locale.LC_ALL, native_str('English_United States.1252')) except Exception: msg = "Could not set locale to English/United States. " + \ "Some date-related tests may fail" warnings.warn(msg) # set matplotlib builtin default settings for testing rcdefaults() if self.style is not None: self.style.__enter__() if MATPLOTLIB_VERSION >= [2, 0, 0]: default_font = 'DejaVu Sans' else: default_font = 'Bitstream Vera Sans' rcParams['font.family'] = default_font with warnings.catch_warnings(record=True) as w: warnings.filterwarnings('always', 'findfont:.*') font_manager.findfont(default_font) if w: warnings.warn('Unable to find the ' + default_font + ' font. ' 'Plotting tests will likely fail.') try: rcParams['text.hinting'] = False except KeyError: warnings.warn("could not set rcParams['text.hinting']") try: rcParams['text.hinting_factor'] = 8 except KeyError: warnings.warn("could not set rcParams['text.hinting_factor']") if self.plt_close_all_enter: import matplotlib.pyplot as plt try: plt.close("all") except Exception: pass return self
def test_evalresp_spline(self): """ evr_spline was based on GPL plotutils, now replaced by LGPL spline library. Unittest for this function. """ # char *evr_spline(int num_points, double *t, double *y, # double tension, double k, # double *xvals_arr, int num_xvals, # double **p_retvals_arr, int *p_num_retvals) clibevresp.evr_spline.argtypes = [ C.c_int, # num_points np.ctypeslib.ndpointer(dtype=np.float64, ndim=1, flags=native_str('C_CONTIGUOUS')), np.ctypeslib.ndpointer(dtype=np.float64, ndim=1, flags=native_str('C_CONTIGUOUS')), C.c_double, # tension C.c_double, # k np.ctypeslib.ndpointer(dtype=np.float64, ndim=1, flags=native_str('C_CONTIGUOUS')), C.c_int, # num_xvals C.POINTER(C.POINTER(C.c_double)), C.POINTER(C.c_int) ] clibevresp.evr_spline.restype = C.c_char_p x = np.arange(1.2, 2.0, .1) n = len(x) y = np.sin(x) xi = x[:-1] + .05 ni = len(xi) p_num_retvals = C.c_int(0) p_retvals_arr = C.POINTER(C.c_double)() res = clibevresp.evr_spline(n, x, y, 0.0, 1.0, xi, ni, C.byref(p_retvals_arr), C.byref(p_num_retvals)) self.assertEqual(res, None) self.assertEqual(ni, p_num_retvals.value) yi = np.array([p_retvals_arr[i] for i in range(ni)]) if False: # visually verify import matplotlib.pyplot as plt plt.plot(x, y, 'bo-', 'Orig values') plt.plot(xi, yi, 'ro-', 'Cubic Spline interpolated values') plt.legend() plt.show() yi_ref = [0.94899576, 0.97572004, 0.9927136, 0.99978309, 0.99686554, 0.98398301, 0.96128491] self.assertTrue(np.allclose(yi, yi_ref, rtol=1e-7, atol=0))
def add_doctests(testsuite, module_name): """ Function to add all available doctests of the module with given name (e.g. "obspy.core") to the given unittest TestSuite. All submodules in the module's root directory are added. Occurring errors are shown as warnings. :type testsuite: unittest.TestSuite :param testsuite: testsuite to which the tests should be added :type module_name: str :param module_name: name of the module of which the tests should be added .. rubric:: Example >>> import unittest >>> suite = unittest.TestSuite() >>> add_doctests(suite, "obspy.core") """ MODULE_NAME = module_name MODULE = __import__(MODULE_NAME, fromlist=[native_str("obspy")]) MODULE_PATH = MODULE.__path__[0] MODULE_PATH_LEN = len(MODULE_PATH) for root, _dirs, files in os.walk(MODULE_PATH): # skip directories without __init__.py if '__init__.py' not in files: continue # skip tests directories if root.endswith('tests'): continue # skip scripts directories if root.endswith('scripts'): continue # skip lib directories if root.endswith('lib'): continue # loop over all files for file in files: # skip if not python source file if not file.endswith('.py'): continue # get module name parts = root[MODULE_PATH_LEN:].split(os.sep)[1:] module_name = ".".join([MODULE_NAME] + parts + [file[:-3]]) try: module = __import__(module_name, fromlist=[native_str("obspy")]) testsuite.addTest(doctest.DocTestSuite(module)) except ValueError: pass
def test_native_str(self): """ Tests whether native_str is really equal to the platform str. """ if PY2: import __builtin__ builtin_str = __builtin__.str else: import builtins builtin_str = builtins.str inputs = [b'blah', u'blah', 'blah'] for s in inputs: self.assertEqual(native_str(s), builtin_str(s)) self.assertTrue(isinstance(native_str(s), builtin_str))
def write_png(arr, filename): """ Custom write_png() function. matplotlib < 1.3 cannot write RGBA png files. Modified from https://stackoverflow.com/a/19174800 """ import zlib import struct buf = arr[::-1, :, :].tostring() height, width, _ = arr.shape # reverse the vertical line order and add null bytes at the start width_byte_4 = width * 4 raw_data = b''.join( b'\x00' + buf[span:span + width_byte_4] for span in range((height - 1) * width * 4, -1, - width_byte_4)) def png_pack(png_tag, data): chunk_head = png_tag + data return (struct.pack(native_str("!I"), len(data)) + chunk_head + struct.pack(native_str("!I"), 0xFFFFFFFF & zlib.crc32(chunk_head))) with open(filename, "wb") as fh: fh.write(b''.join([ b'\x89PNG\r\n\x1a\n', png_pack(b'IHDR', struct.pack(native_str("!2I5B"), width, height, 8, 6, 0, 0, 0)), png_pack(b'IDAT', zlib.compress(raw_data, 9)), png_pack(b'IEND', b'')]))
def load(self, raw_torrent, fields=None, start=False, mkdir=True): if fields is None: fields = {} # First param is empty 'target' params = ['', xmlrpc_client.Binary(raw_torrent)] # Additional fields to set for key, val in fields.items(): # Values must be escaped if within params params.append('d.%s.set=%s' % (key, re.escape(native_str(val)))) if mkdir and 'directory' in fields: result = self._server.execute.throw('', 'mkdir', '-p', fields['directory']) if result != 0: raise xmlrpc_client.Error('Failed creating directory %s' % fields['directory']) # by default rtorrent won't allow calls over 512kb in size. xmlrpc_size = len(xmlrpc_client.dumps(tuple(params), 'raw_start')) + 71680 # Add 70kb for buffer if xmlrpc_size > 524288: prev_size = self._server.network.xmlrpc.size_limit() self._server.network.xmlrpc.size_limit.set('', xmlrpc_size) # Call load method and return the response if start: result = self._server.load.raw_start(*params) else: result = self._server.load.raw(*params) if xmlrpc_size > 524288: self._server.network.xmlrpc.size_limit.set('', prev_size) return result
def _get_blockettes(self): """ Loop over header and try to extract all header values! """ self.blockettes = OrderedDict() cur_blkt_offset = self.fixed_header['First blockette'] # Loop until the beginning of the data is reached. while True: if len(self.blockettes) == \ self.fixed_header["Number of blockettes that follow"]: break # Seek to the offset. self.file.seek(self.record_offset + cur_blkt_offset, 0) # Unpack the first two values. This is always the blockette type # and the beginning of the next blockette. encoding = native_str('%s2H' % self.endian) _tmp = self.file.read(4) try: blkt_type, next_blockette = unpack(encoding, _tmp) except Exception: if len(_tmp) == 0: msg = "Unexpected end of file." raise IOError(msg) raise blkt_type = int(blkt_type) next_blockette = int(next_blockette) self.blockettes[blkt_type] = self._parse_blockette(blkt_type) # Also break the loop if next_blockette is zero. if next_blockette == 0 or next_blockette < 4 or \ next_blockette - 4 < cur_blkt_offset: break cur_blkt_offset = next_blockette
def _run_indexer(options): logging.info("Starting indexer %s:%s ..." % (options.host, options.port)) # initialize crawler service = WaveformIndexer((options.host, options.port), MyHandler) service.log = logging try: # prepare paths if ',' in options.data: paths = options.data.split(',') else: paths = [options.data] paths = service._prepare_paths(paths) if not paths: return # prepare map file if options.mapping_file: with open(options.mapping_file, 'r') as f: data = f.readlines() mappings = parse_mapping_data(data) logging.info("Parsed %d lines from mapping file %s" % (len(data), options.mapping_file)) else: mappings = {} # create file queue and worker processes manager = multiprocessing.Manager() in_queue = manager.dict() work_queue = manager.list() out_queue = manager.list() log_queue = manager.list() # spawn processes for i in range(options.number_of_cpus): args = (i, in_queue, work_queue, out_queue, log_queue, mappings) p = multiprocessing.Process(target=worker, args=args) p.daemon = True p.start() # connect to database engine = create_engine(options.db_uri, encoding=native_str('utf-8'), convert_unicode=True) metadata = Base.metadata # recreate database if options.drop_database: metadata.drop_all(engine, checkfirst=True) metadata.create_all(engine, checkfirst=True) # initialize database + options _session = sessionmaker(bind=engine) service.session = _session service.options = options service.mappings = mappings # set queues service.input_queue = in_queue service.work_queue = work_queue service.output_queue = out_queue service.log_queue = log_queue service.paths = paths service._reset_walker() service._step_walker() service.serve_forever(options.poll_interval) except KeyboardInterrupt: quit() logging.info("Indexer stopped.")
def _run_checker_for_package(checker, package_name, extra_ignore=None): """ Runs the checker function across every Python module in the given package. """ ignore_strings = IGNORE_ERRORS if extra_ignore: ignore_strings += extra_ignore package_path = path_for_import(package_name) for (root, dirs, files) in os.walk(package_path): for f in files: # Ignore migrations. directory = root.split(os.sep)[-1] # Using native_str here avoids the dreaded UnicodeDecodeError # on Py2 with filenames with high-bit characters when # unicode_literals in effect: ext = native_str(".py") if (f == "local_settings.py" or not f.endswith(ext) or directory == "migrations"): continue for warning in checker(os.path.join(root, f)): for ignore in ignore_strings: if ignore in warning: break else: yield warning.replace(package_path, package_name, 1)
def single_request(self, host, handler, request_body, verbose=0): # Add SCGI headers to the request. headers = [('CONTENT_LENGTH', native_str(len(request_body))), ('SCGI', '1')] header = '\x00'.join(['%s\x00%s' % (key, value) for key, value in headers]) + '\x00' header = '%d:%s' % (len(header), header) request_body = '%s,%s' % (header, request_body) sock = None try: if host: parsed_host = urlparse(host) host = parsed_host.hostname port = parsed_host.port addr_info = socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM) sock = socket.socket(*addr_info[0][:3]) sock.connect(addr_info[0][4]) else: sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sock.connect(handler) self.verbose = verbose sock.sendall(request_body.encode()) return self.parse_response(sock.makefile()) finally: if sock: sock.close()
def test_rating(self): """ Test that ratings can be posted and avarage/count are calculated. """ blog_post = BlogPost.objects.create(title="Ratings", user=self._user, status=CONTENT_STATUS_PUBLISHED) if settings.RATINGS_ACCOUNT_REQUIRED: self.client.login(username=self._username, password=self._password) data = RatingForm(None, blog_post).initial for value in settings.RATINGS_RANGE: data["value"] = value response = self.client.post(reverse("rating"), data=data) # Django doesn't seem to support unicode cookie keys correctly on # Python 2. See https://code.djangoproject.com/ticket/19802 response.delete_cookie(native_str("mezzanine-rating")) blog_post = BlogPost.objects.get(id=blog_post.id) count = len(settings.RATINGS_RANGE) _sum = sum(settings.RATINGS_RANGE) average = _sum / count if settings.RATINGS_ACCOUNT_REQUIRED: self.assertEqual(blog_post.rating_count, 1) self.assertEqual(blog_post.rating_sum, settings.RATINGS_RANGE[-1]) self.assertEqual(blog_post.rating_average, settings.RATINGS_RANGE[-1] / 1) else: self.assertEqual(blog_post.rating_count, count) self.assertEqual(blog_post.rating_sum, _sum) self.assertEqual(blog_post.rating_average, average)
def getExampleFile(filename): """ Function to find the absolute path of a test data file The ObsPy modules are installed to a custom installation directory. That is the path cannot be predicted. This functions searches for all installed ObsPy modules and checks whether the file is in any of the "tests/data" subdirectories. :param filename: A test file name to which the path should be returned. :return: Full path to file. .. rubric:: Example >>> getExampleFile('slist.ascii') # doctest: +SKIP /custom/path/to/obspy/core/tests/data/slist.ascii >>> getExampleFile('does.not.exists') # doctest: +ELLIPSIS Traceback (most recent call last): ... OSError: Could not find file does.not.exists ... """ for module in ALL_MODULES: try: mod = __import__("obspy.%s.tests" % module, fromlist=[native_str("obspy")]) except ImportError: continue file = os.path.join(mod.__path__[0], "data", filename) if os.path.isfile(file): return file msg = "Could not find file %s in tests/data directory " % filename + \ "of ObsPy modules" raise OSError(msg)
def _enforce_telegram_plugin_ver(): if telegram is None: raise plugin.PluginWarning('missing python-telegram-bot pkg') elif not hasattr(telegram, str('__version__')): raise plugin.PluginWarning('invalid or old python-telegram-bot pkg') elif LooseVersion(telegram.__version__) < native_str(_MIN_TELEGRAM_VER): raise plugin.PluginWarning('old python-telegram-bot ({0})'.format(telegram.__version__))
def _load_cache(self, fname): if not exists(fname): raise Exception("Cache file does not exist.") # TODO: Python 3 workaround numpy issue a = np.loadtxt(native_str(fname)) ra = a[:,0] vv0 = a[:,1] vv2 = a[:,2] vv4 = a[:,3] if not self._vv_only: if(a.shape[1] != 7): raise Exception("Cache file has wrong number of columns.") dd0 = a[:,4] dv0 = a[:,5] dv2 = a[:,6] self._vv0i = cs.Interpolater(ra, vv0) self._vv2i = cs.Interpolater(ra, vv2) self._vv4i = cs.Interpolater(ra, vv4) if not self._vv_only: self._dd0i = cs.Interpolater(ra, dd0) self._dv0i = cs.Interpolater(ra, dv0) self._dv2i = cs.Interpolater(ra, dv2) self._cached = True
def compare_images(expected, actual, tol): """ Custom version of :func:`matplotlib.testing.compare.compare_images`. This enable ObsPy to have the same image comparison metric across matplotlib versions. Furthermore nose is no longer a test dependency of ObsPy. In contrast to the matplotlib version this one only works with png files. Fully transparent pixels will have their color set to white as the RGB values of fully transparent pixels change depending on the matplotlib version. Additionally this version uses a straight RMSE definition instead of the binned one of matplotlib. :param expected: The filename of the expected png file. :type expected: str :param actual: The filename of the actual png file. :type actual: str :param tol: The tolerance (a color value difference, where 255 is the maximal difference). The test fails if the average pixel difference is greater than this value. :type tol: float """ import matplotlib.image if not os.path.exists(actual): msg = "Output image %s does not exist." % actual raise Exception(msg) if os.stat(actual).st_size == 0: msg = "Output image file %s is empty." % actual raise Exception(msg) if not os.path.exists(expected): raise IOError('Baseline image %r does not exist.' % expected) # Open the images. Will be opened as RGBA as float32 ranging from 0 to 1. expected_image = matplotlib.image.imread(native_str(expected)) actual_image = matplotlib.image.imread(native_str(actual)) if expected_image.shape != actual_image.shape: raise ImageComparisonException( "The shape of the received image %s is not equal to the expected " "shape %s." % (str(actual_image.shape), str(expected_image.shape))) # Set the "color" of fully transparent pixels to white. This avoids the # issue of different "colors" for transparent pixels. expected_image[expected_image[..., 3] <= 0.0035] = [1.0, 1.0, 1.0, 0.0] actual_image[actual_image[..., 3] <= 0.0035] = [1.0, 1.0, 1.0, 0.0] # This deviates a bit from the matplotlib version and just calculates # the root mean square error of all pixel values without any other fancy # considerations. It also uses the alpha channel of the images. Scaled # by 255. rms = np.sqrt( np.sum((255.0 * (expected_image - actual_image))**2) / float(expected_image.size)) base, ext = os.path.splitext(actual) diff_image = '%s-%s%s' % (base, 'failed-diff', ext) if rms <= tol: if os.path.exists(diff_image): os.unlink(diff_image) return None # Save diff image, expand differences in luminance domain abs_diff_image = np.abs(expected_image - actual_image) abs_diff_image *= 10.0 save_image_np = np.clip(abs_diff_image, 0.0, 1.0) # Hard-code the alpha channel to fully solid save_image_np[:, :, 3] = 1.0 write_png(np.uint8(save_image_np * 255.0), diff_image) return dict(rms=rms, expected=str(expected), actual=str(actual), diff=str(diff_image), tol=tol)
can optionally filter the data. **Calculation Tools** * :meth:`~obspy.clients.iris.client.Client.traveltime()` - calculates travel-times for seismic phases using a 1-D spherical Earth model. * :meth:`~obspy.clients.iris.client.Client.distaz()` - calculate the distance and azimuth between two points on a sphere. * :meth:`~obspy.clients.iris.client.Client.flinnengdahl()` - converts a latitude, longitude pair into either a Flinn-Engdahl seismic region code or region name. Please see the documentation for each method for further information and examples to retrieve various data from the IRIS DMC. """ from __future__ import (absolute_import, division, print_function, unicode_literals) from future.builtins import * # NOQA from future.utils import native_str from .client import Client # NOQA __all__ = [native_str("Client")] if __name__ == '__main__': import doctest doctest.testmod(exclude_empty=True)
def _add_inventory_layer(data_source, inventory): """ :type data_source: :class:`osgeo.ogr.DataSource`. :param data_source: OGR data source the layer is added to. :type inventory: :class:`~obspy.core.inventory.Inventory` :param inventory: Inventory data to add as a new layer. """ if not has_GDAL: raise ImportError(IMPORTERROR_MSG) # [name, type, width, precision] # field name is 10 chars max # ESRI shapefile attributes are stored in dbf files, which can not # store datetimes, only dates, see: # http://www.gdal.org/drv_shapefile.html # use POSIX timestamp for exact origin time, set time of first pick # for events with no origin field_definitions = [ ["Network", ogr.OFTString, 20, None], ["Station", ogr.OFTString, 20, None], ["Longitude", ogr.OFTReal, 16, 10], ["Latitude", ogr.OFTReal, 16, 10], ["Elevation", ogr.OFTReal, 9, 3], ["StartDate", ogr.OFTDate, None, None], ["EndDate", ogr.OFTDate, None, None], ["Channels", ogr.OFTString, 254, None], ] layer = _create_layer(data_source, "stations", field_definitions) layer_definition = layer.GetLayerDefn() for net in inventory: for sta in net: channel_list = ",".join( ["%s.%s" % (cha.location_code, cha.code) for cha in sta]) feature = ogr.Feature(layer_definition) try: # setting fields with `None` results in values of `0.000` # need to really omit setting values if they are `None` if net.code is not None: feature.SetField(native_str("Network"), native_str(net.code)) if sta.code is not None: feature.SetField(native_str("Station"), native_str(sta.code)) if sta.latitude is not None: feature.SetField(native_str("Latitude"), sta.latitude) if sta.longitude is not None: feature.SetField(native_str("Longitude"), sta.longitude) if sta.elevation is not None: feature.SetField(native_str("Elevation"), sta.elevation) if sta.start_date is not None: date = sta.start_date # ESRI shapefile attributes are stored in dbf files, which # can not store datetimes, only dates. We still need to use # the GDAL API with precision up to seconds (aiming at # other output drivers of GDAL; `100` stands for GMT) feature.SetField(native_str("StartDate"), date.year, date.month, date.day, date.hour, date.minute, date.second, 100) if sta.end_date is not None: date = sta.end_date # ESRI shapefile attributes are stored in dbf files, which # can not store datetimes, only dates. We still need to use # the GDAL API with precision up to seconds (aiming at # other output drivers of GDAL; `100` stands for GMT) feature.SetField(native_str("StartDate"), date.year, date.month, date.day, date.hour, date.minute, date.second, 100) if channel_list: feature.SetField(native_str("Channels"), native_str(channel_list)) if sta.latitude is not None and sta.longitude is not None: point = ogr.Geometry(ogr.wkbPoint) point.AddPoint(sta.longitude, sta.latitude) feature.SetGeometry(point) layer.CreateFeature(feature) finally: # Destroy the feature to free resources feature.Destroy()
from __future__ import (absolute_import, division, print_function, unicode_literals) from future.builtins import * # NOQA from future.utils import native_str import os import wave import numpy as np from obspy import Stream, Trace # WAVE data format is unsigned char up to 8bit, and signed int # for the remaining. WIDTH2DTYPE = { 1: native_str('<u1'), # unsigned char 2: native_str('<i2'), # signed short int 4: native_str('<i4'), # signed int (int32) } def _is_wav(filename): """ Checks whether a file is a audio WAV file or not. :type filename: str :param filename: Name of the audio WAV file to be checked. :rtype: bool :return: ``True`` if a WAV file. .. rubric:: Example
def delete(self, info_hash): return self._server.d.erase(native_str(info_hash))
def iso_utc(now=None, sep='_', t=time.time): if now is None: now = t() sep = native_str(sep) # Python 2 doesn't allow unicode input to isoformat return datetime.datetime.utcfromtimestamp(now).isoformat(sep)
def stop(self, info_hash): self._server.d.stop(info_hash) return self._server.d.close(native_str(info_hash))
def time_multi_normxcorr(templates, stream, pads, threaded=False, *args, **kwargs): """ Compute cross-correlations in the time-domain using C routine. :param templates: 2D Array of templates :type templates: np.ndarray :param stream: 1D array of continuous data :type stream: np.ndarray :param pads: List of ints of pad lengths in the same order as templates :type pads: list :param threaded: Whether to use the threaded routine or not :type threaded: bool :return: np.ndarray of cross-correlations :return: np.ndarray channels used """ used_chans = ~np.isnan(templates).any(axis=1) utilslib = _load_cdll('libutils') argtypes = [ np.ctypeslib.ndpointer(dtype=np.float32, ndim=1, flags=native_str('C_CONTIGUOUS')), ctypes.c_int, ctypes.c_int, np.ctypeslib.ndpointer(dtype=np.float32, ndim=1, flags=native_str('C_CONTIGUOUS')), ctypes.c_int, np.ctypeslib.ndpointer(dtype=np.float32, ndim=1, flags=native_str('C_CONTIGUOUS'))] restype = ctypes.c_int if threaded: func = utilslib.multi_normxcorr_time_threaded argtypes.append(ctypes.c_int) else: func = utilslib.multi_normxcorr_time func.argtypes = argtypes func.restype = restype # Need to de-mean everything templates_means = templates.mean(axis=1).astype(np.float32)[:, np.newaxis] stream_mean = stream.mean().astype(np.float32) templates = templates.astype(np.float32) - templates_means stream = stream.astype(np.float32) - stream_mean template_len = templates.shape[1] n_templates = templates.shape[0] image_len = stream.shape[0] ccc = np.ascontiguousarray( np.empty((image_len - template_len + 1) * n_templates), np.float32) t_array = np.ascontiguousarray(templates.flatten(), np.float32) time_args = [t_array, template_len, n_templates, np.ascontiguousarray(stream, np.float32), image_len, ccc] if threaded: time_args.append(kwargs.get('cores', cpu_count())) func(*time_args) ccc[np.isnan(ccc)] = 0.0 ccc = ccc.reshape((n_templates, image_len - template_len + 1)) for i in range(len(pads)): ccc[i] = np.append(ccc[i], np.zeros(pads[i]))[pads[i]:] templates += templates_means stream += stream_mean return ccc, used_chans
def templates_max_similarity(st, time, streams_templates): """ Compares all event templates in the streams_templates list of streams against the given stream around the time of the suspected event. The stream that is being checked has to include all trace ids that are included in template events. One component streams can be checked as well as multiple components simultaneously. In case of multiple components it is made sure, that all three components are shifted together. The traces in any stream need to have a reasonable common starting time. The stream to check should have some additional data to left/right of suspected event, the event template streams should be cut to the portion of the event that should be compared. Also see :func:`obspy.signal.trigger.coincidence_trigger` and the corresponding example in the `Trigger/Picker Tutorial <https://tutorial.obspy.org/code_snippets/trigger_tutorial.html>`_. - computes cross correlation on each component (one stream serves as template, one as a longer search stream) - stack all three and determine best shift in stack - normalization is a bit problematic so compute the correlation coefficient afterwards for the best shift to make sure the result is between 0 and 1. >>> from obspy import read, UTCDateTime >>> import numpy as np >>> np.random.seed(123) # make test reproducible >>> st = read() >>> t = UTCDateTime(2009, 8, 24, 0, 20, 7, 700000) >>> templ = st.copy().slice(t, t+5) >>> for tr in templ: ... tr.data += np.random.random(len(tr)) * tr.data.max() * 0.5 >>> print(templates_max_similarity(st, t, [templ])) 0.922536411468 :param time: Time around which is checked for a similarity. Cross correlation shifts of around template event length are checked. :type time: :class:`~obspy.core.utcdatetime.UTCDateTime` :param st: One or multi-component Stream to check against event templates. Should have additional data left/right of suspected event (around half the length of template events). :type st: :class:`~obspy.core.stream.Stream` :param streams_templates: List of streams with template events to check for waveform similarity. Each template has to include data for all channels present in stream to check. :type streams_templates: list of :class:`~obspy.core.stream.Stream` :returns: Best correlation coefficient obtained by the comparison against all template events (0 to 1). """ values = [] for st_tmpl in streams_templates: ids = [tr.id for tr in st_tmpl] duration = st_tmpl[0].stats.endtime - st_tmpl[0].stats.starttime st_ = st.slice(time - (duration * 0.5), time + (duration * 1.5)) cc = None for id_ in reversed(ids): if not st_.select(id=id_): msg = "Skipping trace %s in template correlation " + \ "(not present in stream to check)." warnings.warn(msg % id_) ids.remove(id_) # determine best (combined) shift of multi-component data for id_ in ids: tr1 = st_.select(id=id_)[0] tr2 = st_tmpl.select(id=id_)[0] if len(tr1) > len(tr2): data_short = tr2.data data_long = tr1.data else: data_short = tr1.data data_long = tr2.data data_short = (data_short - data_short.mean()) / data_short.std() data_long = (data_long - data_long.mean()) / data_long.std() tmp = np.correlate(data_long, data_short, native_str("valid")) try: cc += tmp except TypeError: cc = tmp except ValueError: cc = None break if cc is None: msg = "Skipping template(s) for station %s due to problems in " + \ "three component correlation (gappy traces?)" warnings.warn(msg % st_tmpl[0].stats.station) break ind = cc.argmax() ind2 = ind + len(data_short) coef = 0.0 # determine correlation coefficient of best shift as the mean of all # components for id_ in ids: tr1 = st_.select(id=id_)[0] tr2 = st_tmpl.select(id=id_)[0] if len(tr1) > len(tr2): data_short = tr2.data data_long = tr1.data else: data_short = tr1.data data_long = tr2.data coef += np.corrcoef(data_short, data_long[ind:ind2])[0, 1] coef /= len(ids) values.append(coef) if values: return max(values) else: return 0
import numpy as np import sys import os import warnings # Get the system byte order. BYTEORDER = sys.byteorder if BYTEORDER == 'little': BYTEORDER = '<' else: BYTEORDER = '>' clibsegy.ibm2ieee.argtypes = [ np.ctypeslib.ndpointer(dtype=np.float32, ndim=1, flags=native_str('C_CONTIGUOUS')), C.c_int ] clibsegy.ibm2ieee.restype = C.c_void_p def unpack_4byte_IBM(file, count, endian='>'): """ Unpacks 4 byte IBM floating points. """ # Read as 4 byte integer so bit shifting works. data = np.fromstring(file.read(count * 4), dtype=np.float32) # Swap the byte order if necessary. if BYTEORDER != endian: data = data.byteswap() length = len(data) # Call the C code which transforms the data inplace.
def test_issue296(self): """ Tests issue #296. """ with NamedTemporaryFile() as tf: tempfile = tf.name # 1 - transform to np.float64 values st = read() for tr in st: tr.data = tr.data.astype(np.float64) # write a single trace automatically detecting encoding st[0].write(tempfile, format="MSEED") # write a single trace automatically detecting encoding st.write(tempfile, format="MSEED") # write a single trace with encoding 5 st[0].write(tempfile, format="MSEED", encoding=5) # write a single trace with encoding 5 st.write(tempfile, format="MSEED", encoding=5) # 2 - transform to np.float32 values st = read() for tr in st: tr.data = tr.data.astype(np.float32) # write a single trace automatically detecting encoding st[0].write(tempfile, format="MSEED") # write a single trace automatically detecting encoding st.write(tempfile, format="MSEED") # write a single trace with encoding 4 st[0].write(tempfile, format="MSEED", encoding=4) # write a single trace with encoding 4 st.write(tempfile, format="MSEED", encoding=4) # 3 - transform to np.int32 values st = read() for tr in st: tr.data = tr.data.astype(np.int32) # write a single trace automatically detecting encoding st[0].write(tempfile, format="MSEED") # write a single trace automatically detecting encoding st.write(tempfile, format="MSEED") # write a single trace with encoding 3 st[0].write(tempfile, format="MSEED", encoding=3) # write the whole stream with encoding 3 st.write(tempfile, format="MSEED", encoding=3) # write a single trace with encoding 10 st[0].write(tempfile, format="MSEED", encoding=10) # write the whole stream with encoding 10 st.write(tempfile, format="MSEED", encoding=10) # write a single trace with encoding 11 st[0].write(tempfile, format="MSEED", encoding=11) # write the whole stream with encoding 11 st.write(tempfile, format="MSEED", encoding=11) # 4 - transform to np.int16 values st = read() for tr in st: tr.data = tr.data.astype(np.int16) # write a single trace automatically detecting encoding st[0].write(tempfile, format="MSEED") # write a single trace automatically detecting encoding st.write(tempfile, format="MSEED") # write a single trace with encoding 1 st[0].write(tempfile, format="MSEED", encoding=1) # write the whole stream with encoding 1 st.write(tempfile, format="MSEED", encoding=1) # 5 - transform to ASCII values st = read() for tr in st: tr.data = tr.data.astype(native_str('|S1')) # write a single trace automatically detecting encoding st[0].write(tempfile, format="MSEED") # write a single trace automatically detecting encoding st.write(tempfile, format="MSEED") # write a single trace with encoding 0 st[0].write(tempfile, format="MSEED", encoding=0) # write the whole stream with encoding 0 st.write(tempfile, format="MSEED", encoding=0)
def _add_catalog_layer(data_source, catalog): """ :type data_source: :class:`osgeo.ogr.DataSource`. :param data_source: OGR data source the layer is added to. :type catalog: :class:`~obspy.core.event.Catalog` :param catalog: Event data to add as a new layer. """ if not has_GDAL: raise ImportError(IMPORTERROR_MSG) # [name, type, width, precision] # field name is 10 chars max # ESRI shapefile attributes are stored in dbf files, which can not # store datetimes, only dates, see: # http://www.gdal.org/drv_shapefile.html # use POSIX timestamp for exact origin time, set time of first pick # for events with no origin field_definitions = [ ["EventID", ogr.OFTString, 100, None], ["OriginID", ogr.OFTString, 100, None], ["MagID", ogr.OFTString, 100, None], ["Date", ogr.OFTDate, None, None], ["OriginTime", ogr.OFTReal, 20, 6], ["FirstPick", ogr.OFTReal, 20, 6], ["Longitude", ogr.OFTReal, 16, 10], ["Latitude", ogr.OFTReal, 16, 10], ["Depth", ogr.OFTReal, 8, 3], ["Magnitude", ogr.OFTReal, 8, 3], ] layer = _create_layer(data_source, "earthquakes", field_definitions) layer_definition = layer.GetLayerDefn() for event in catalog: # try to use preferred origin/magnitude, fall back to first or use # empty one with `None` values in it origin = (event.preferred_origin() or event.origins and event.origins[0] or Origin(force_resource_id=False)) magnitude = (event.preferred_magnitude() or event.magnitudes and event.magnitudes[0] or Magnitude(force_resource_id=False)) t_origin = origin.time pick_times = [ pick.time for pick in event.picks if pick.time is not None ] t_pick = pick_times and min(pick_times) or None date = t_origin or t_pick feature = ogr.Feature(layer_definition) try: # setting fields with `None` results in values of `0.000` # need to really omit setting values if they are `None` if event.resource_id is not None: feature.SetField(native_str("EventID"), native_str(event.resource_id)) if origin.resource_id is not None: feature.SetField(native_str("OriginID"), native_str(origin.resource_id)) if t_origin is not None: # Use timestamp for exact timing feature.SetField(native_str("OriginTime"), t_origin.timestamp) if t_pick is not None: # Use timestamp for exact timing feature.SetField(native_str("FirstPick"), t_pick.timestamp) if date is not None: # ESRI shapefile attributes are stored in dbf files, which can # not store datetimes, only dates. We still need to use the # GDAL API with precision up to seconds (aiming at other output # drivers of GDAL; `100` stands for GMT) feature.SetField(native_str("Date"), date.year, date.month, date.day, date.hour, date.minute, date.second, 100) if origin.latitude is not None: feature.SetField(native_str("Latitude"), origin.latitude) if origin.longitude is not None: feature.SetField(native_str("Longitude"), origin.longitude) if origin.depth is not None: feature.SetField(native_str("Depth"), origin.depth / 1e3) if magnitude.mag is not None: feature.SetField(native_str("Magnitude"), magnitude.mag) if magnitude.resource_id is not None: feature.SetField(native_str("MagID"), native_str(magnitude.resource_id)) if origin.latitude is not None and origin.longitude is not None: point = ogr.Geometry(ogr.wkbPoint) point.AddPoint(origin.longitude, origin.latitude) feature.SetGeometry(point) layer.CreateFeature(feature) finally: # Destroy the feature to free resources feature.Destroy()
def _read_q(filename, headonly=False, data_directory=None, byteorder='=', **kwargs): # @UnusedVariable """ Reads a Seismic Handler Q file and returns an ObsPy Stream object. .. warning:: This function should NOT be called directly, it registers via the ObsPy :func:`~obspy.core.stream.read` function, call this instead. :type filename: str :param filename: Q header file to be read. Must have a `QHD` file extension. :type headonly: bool, optional :param headonly: If set to True, read only the head. This is most useful for scanning available data in huge (temporary) data sets. :type data_directory: str, optional :param data_directory: Data directory where the corresponding QBN file can be found. :type byteorder: str, optional :param byteorder: Enforce byte order for data file. This is important for Q files written in older versions of Seismic Handler, which don't explicit state the `BYTEORDER` flag within the header file. Can be little endian (``'<'``), big endian (``'>'``), or native byte order (``'='``). Defaults to ``'='``. :rtype: :class:`~obspy.core.stream.Stream` :return: A ObsPy Stream object. Q files consists of two files per data set: * a ASCII header file with file extension `QHD` and the * binary data file with file extension `QBN`. The read method only accepts header files for the ``filename`` parameter. ObsPy assumes that the corresponding data file is within the same directory if the ``data_directory`` parameter is not set. Otherwise it will search in the given ``data_directory`` for a file with the `QBN` file extension. This function should NOT be called directly, it registers via the ObsPy :func:`~obspy.core.stream.read` function, call this instead. .. rubric:: Example >>> from obspy import read >>> st = read("/path/to/QFILE-TEST.QHD") >>> st #doctest: +ELLIPSIS <obspy.core.stream.Stream object at 0x...> >>> print(st) # doctest: +ELLIPSIS 3 Trace(s) in Stream: .TEST..BHN | 2009-10-01T12:46:01.000000Z - ... | 20.0 Hz, 801 samples .TEST..BHE | 2009-10-01T12:46:01.000000Z - ... | 20.0 Hz, 801 samples .WET..HHZ | 2010-01-01T01:01:05.999000Z - ... | 100.0 Hz, 4001 samples """ if not headonly: if not data_directory: data_file = os.path.splitext(filename)[0] + '.QBN' else: data_file = os.path.basename(os.path.splitext(filename)[0]) data_file = os.path.join(data_directory, data_file + '.QBN') if not os.path.isfile(data_file): msg = "Can't find corresponding QBN file at %s." raise IOError(msg % data_file) fh_data = open(data_file, 'rb') # loop through read header file with open(filename, 'rt') as fh: lines = fh.read().splitlines() # number of comment lines cmtlines = int(lines[0][5:7]) # trace lines traces = {} i = -1 id = '' for line in lines[cmtlines:]: cid = int(line[0:2]) if cid != id: id = cid i += 1 traces.setdefault(i, '') traces[i] += line[3:] # create stream object stream = Stream() for id in sorted(traces.keys()): # fetch headers header = {} header['sh'] = { "FROMQ": True, "FILE": os.path.splitext(os.path.split(filename)[1])[0], } channel = ['', '', ''] npts = 0 for item in traces[id].split('~'): key = item.lstrip()[0:4] value = item.lstrip()[5:] if key == 'L001': npts = header['npts'] = int(value) elif key == 'L000': continue elif key == 'R000': header['delta'] = float(value) elif key == 'R026': header['calib'] = float(value) elif key == 'S001': header['station'] = value elif key == 'C000' and value: channel[2] = value[0] elif key == 'C001' and value: channel[0] = value[0] elif key == 'C002' and value: channel[1] = value[0] elif key == 'C003': if value == '<' or value == '>': byteorder = header['sh']['BYTEORDER'] = value elif key == 'S021': # 01-JAN-2009_01:01:01.0 # 1-OCT-2009_12:46:01.000 header['starttime'] = to_utcdatetime(value) elif key == 'S022': header['sh']['P-ONSET'] = to_utcdatetime(value) elif key == 'S023': header['sh']['S-ONSET'] = to_utcdatetime(value) elif key == 'S024': header['sh']['ORIGIN'] = to_utcdatetime(value) elif key: key = INVERTED_SH_IDX.get(key, key) if key in SH_KEYS_INT: header['sh'][key] = int(value) elif key in SH_KEYS_FLOAT: header['sh'][key] = float(value) else: header['sh'][key] = value # set channel code header['channel'] = ''.join(channel) # remember record number header['sh']['RECNO'] = len(stream) + 1 if headonly: # skip data stream.append(Trace(header=header)) else: if not npts: stream.append(Trace(header=header)) continue # read data data = fh_data.read(npts * 4) dtype = native_str(byteorder + 'f4') data = np.fromstring(data, dtype=dtype) # convert to system byte order data = np.require(data, native_str('=f4')) stream.append(Trace(data=data, header=header)) if not headonly: fh_data.close() return stream
unicode_literals) from future.builtins import * # NOQA from future.utils import native_str import warnings from obspy.core.util.base import SCIPY_VERSION # Convenience imports. from .mass_downloader import MassDownloader # NOQA from .restrictions import Restrictions # NOQA from .domain import ( Domain, RectangularDomain, # NOQA CircularDomain, GlobalDomain) # NOQA __all__ = [ native_str(i) for i in ('MassDownloader', 'Restrictions', 'Domain', 'RectangularDomain', 'CircularDomain', 'GlobalDomain') ] if SCIPY_VERSION < [0, 12]: msg = ('At least some parts of FDSN Mass downloader might not ' 'work with old scipy versions <0.12.0 (installed: {})') warnings.warn(msg.format(SCIPY_VERSION)) if __name__ == '__main__': import doctest doctest.testmod(exclude_empty=True)
def test_poll_now_and_retry_response_missing_key(self): for endpoint in '/poll-now', '/retry': for body in {}, {'key': self.responses[0].key.urlsafe()}: # hasn't been stored resp = app.application.get_response(endpoint, method='POST', body=native_str(urllib.parse.urlencode(body))) self.assertEquals(400, resp.status_int)
from future.utils import PY2, native_str # don't change order from obspy.core.utcdatetime import UTCDateTime # NOQA from obspy.core.util import _get_version_string __version__ = _get_version_string(abbrev=10) from obspy.core.trace import Trace # NOQA from obspy.core.stream import Stream, read from obspy.core.event import read_events, Catalog from obspy.core.inventory import read_inventory, Inventory # NOQA __all__ = [ "UTCDateTime", "Trace", "__version__", "Stream", "read", "read_events", "Catalog", "read_inventory" ] __all__ = [native_str(i) for i in __all__] # insert supported read/write format plugin lists dynamically in docstrings from obspy.core.util.base import make_format_plugin_table read.__doc__ = \ read.__doc__ % make_format_plugin_table("waveform", "read", numspaces=4) read_events.__doc__ = \ read_events.__doc__ % make_format_plugin_table("event", "read", numspaces=4) if PY2: Stream.write.im_func.func_doc = \ Stream.write.__doc__ % make_format_plugin_table("waveform", "write", numspaces=8) Catalog.write.im_func.func_doc = \ Catalog.write.__doc__ % make_format_plugin_table("event", "write",
def _write_mseed(stream, filename, encoding=None, reclen=None, byteorder=None, sequence_number=None, flush=True, verbose=0, **_kwargs): """ Write Mini-SEED file from a Stream object. .. warning:: This function should NOT be called directly, it registers via the the :meth:`~obspy.core.stream.Stream.write` method of an ObsPy :class:`~obspy.core.stream.Stream` object, call this instead. :type stream: :class:`~obspy.core.stream.Stream` :param stream: A Stream object. :type filename: str :param filename: Name of the output file or a file-like object. :type encoding: int or str, optional :param encoding: Should be set to one of the following supported Mini-SEED data encoding formats: ``ASCII`` (``0``)*, ``INT16`` (``1``), ``INT32`` (``3``), ``FLOAT32`` (``4``)*, ``FLOAT64`` (``5``)*, ``STEIM1`` (``10``) and ``STEIM2`` (``11``)*. If no encoding is given it will be derived from the dtype of the data and the appropriate default encoding (depicted with an asterix) will be chosen. :type reclen: int, optional :param reclen: Should be set to the desired data record length in bytes which must be expressible as 2 raised to the power of X where X is between (and including) 8 to 20. Defaults to 4096 :type byteorder: int or str, optional :param byteorder: Must be either ``0`` or ``'<'`` for LSBF or little-endian, ``1`` or ``'>'`` for MBF or big-endian. ``'='`` is the native byte order. If ``-1`` it will be passed directly to libmseed which will also default it to big endian. Defaults to big endian. :type sequence_number: int, optional :param sequence_number: Must be an integer ranging between 1 and 999999. Represents the sequence count of the first record of each Trace. Defaults to 1. :type flush: bool, optional :param flush: If ``True``, all data will be packed into records. If ``False`` new records will only be created when there is enough data to completely fill a record. Be careful with this. If in doubt, choose ``True`` which is also the default value. :type verbose: int, optional :param verbose: Controls verbosity, a value of ``0`` will result in no diagnostic output. .. note:: The ``reclen``, ``encoding``, ``byteorder`` and ``sequence_count`` keyword arguments can be set in the ``stats.mseed`` of each :class:`~obspy.core.trace.Trace` as well as ``kwargs`` of this function. If both are given the ``kwargs`` will be used. The ``stats.mseed.blkt1001.timing_quality`` value will also be written if it is set. The ``stats.mseed.blkt1001.timing_quality`` value will also be written if it is set. .. rubric:: Example >>> from obspy import read >>> st = read() >>> st.write('filename.mseed', format='MSEED') # doctest: +SKIP """ # Map flush and verbose flags. if flush: flush = 1 else: flush = 0 if not verbose: verbose = 0 if verbose is True: verbose = 1 # Some sanity checks for the keyword arguments. if reclen is not None and reclen not in VALID_RECORD_LENGTHS: msg = 'Invalid record length. The record length must be a value\n' + \ 'of 2 to the power of X where 8 <= X <= 20.' raise ValueError(msg) if byteorder is not None and byteorder not in [0, 1, -1]: if byteorder == '=': byteorder = NATIVE_BYTEORDER # If not elif because NATIVE_BYTEORDER is '<' or '>'. if byteorder == '<': byteorder = 0 elif byteorder == '>': byteorder = 1 else: msg = "Invalid byte order. It must be either '<', '>', '=', " + \ "0, 1 or -1" raise ValueError(msg) if encoding is not None: encoding = util._convert_and_check_encoding_for_writing(encoding) if sequence_number is not None: # Check sequence number type try: sequence_number = int(sequence_number) # Check sequence number value if sequence_number < 1 or sequence_number > 999999: raise ValueError("Sequence number out of range. It must be " + " between 1 and 999999.") except (TypeError, ValueError): msg = "Invalid sequence number. It must be an integer ranging " +\ "from 1 to 999999." raise ValueError(msg) trace_attributes = [] use_blkt_1001 = False # The data might need to be modified. To not modify the input data keep # references of which data to finally write. trace_data = [] # Loop over every trace and figure out the correct settings. for _i, trace in enumerate(stream): # Create temporary dict for storing information while writing. trace_attr = {} trace_attributes.append(trace_attr) # Figure out whether or not to use Blockette 1001. This check is done # once to ensure that Blockette 1001 is either written for every record # in the file or for none. It checks the starttime, the sampling rate # and the timing quality. If starttime or sampling rate has a precision # of more than 100 microseconds, or if timing quality is set, \ # Blockette 1001 will be written for every record. starttime = util._convert_datetime_to_mstime(trace.stats.starttime) if starttime % 100 != 0 or \ (1.0 / trace.stats.sampling_rate * HPTMODULUS) % 100 != 0: use_blkt_1001 = True if hasattr(trace.stats, 'mseed') and \ hasattr(trace.stats['mseed'], 'blkt1001') and \ hasattr(trace.stats['mseed']['blkt1001'], 'timing_quality'): timing_quality = trace.stats['mseed']['blkt1001']['timing_quality'] # Check timing quality type try: timing_quality = int(timing_quality) if timing_quality < 0 or timing_quality > 100: raise ValueError("Timing quality out of range. It must be " "between 0 and 100.") except ValueError: msg = "Invalid timing quality in Stream[%i].stats." % _i + \ "mseed.timing_quality. It must be an integer ranging" + \ " from 0 to 100" raise ValueError(msg) trace_attr['timing_quality'] = timing_quality use_blkt_1001 = True else: trace_attr['timing_quality'] = timing_quality = 0 if sequence_number is not None: trace_attr['sequence_number'] = sequence_number elif hasattr(trace.stats, 'mseed') and \ hasattr(trace.stats['mseed'], 'sequence_number'): sequence_number = trace.stats['mseed']['sequence_number'] # Check sequence number type try: sequence_number = int(sequence_number) # Check sequence number value if sequence_number < 1 or sequence_number > 999999: raise ValueError("Sequence number out of range in " + "Stream[%i].stats. It must be between " + "1 and 999999.") except (TypeError, ValueError): msg = "Invalid sequence number in Stream[%i].stats." % _i +\ "mseed.sequence_number. It must be an integer ranging" +\ " from 1 to 999999." raise ValueError(msg) trace_attr['sequence_number'] = sequence_number else: trace_attr['sequence_number'] = sequence_number = 1 # Set data quality to indeterminate (= D) if it is not already set. try: trace_attr['dataquality'] = \ trace.stats['mseed']['dataquality'].upper() except Exception: trace_attr['dataquality'] = 'D' # Sanity check for the dataquality to get a nice Python exception # instead of a C error. if trace_attr['dataquality'] not in ['D', 'R', 'Q', 'M']: msg = 'Invalid dataquality in Stream[%i].stats' % _i + \ '.mseed.dataquality\n' + \ 'The dataquality for Mini-SEED must be either D, R, Q ' + \ 'or M. See the SEED manual for further information.' raise ValueError(msg) # Check that data is of the right type. if not isinstance(trace.data, np.ndarray): msg = "Unsupported data type %s" % type(trace.data) + \ " for Stream[%i].data." % _i raise ValueError(msg) # Check if ndarray is contiguous (see #192, #193) if not trace.data.flags.c_contiguous: msg = "Detected non contiguous data array in Stream[%i]" % _i + \ ".data. Trying to fix array." warnings.warn(msg) trace.data = np.ascontiguousarray(trace.data) # Handle the record length. if reclen is not None: trace_attr['reclen'] = reclen elif hasattr(trace.stats, 'mseed') and \ hasattr(trace.stats.mseed, 'record_length'): if trace.stats.mseed.record_length in VALID_RECORD_LENGTHS: trace_attr['reclen'] = trace.stats.mseed.record_length else: msg = 'Invalid record length in Stream[%i].stats.' % _i + \ 'mseed.reclen.\nThe record length must be a value ' + \ 'of 2 to the power of X where 8 <= X <= 20.' raise ValueError(msg) else: trace_attr['reclen'] = 4096 # Handle the byte order. if byteorder is not None: trace_attr['byteorder'] = byteorder elif hasattr(trace.stats, 'mseed') and \ hasattr(trace.stats.mseed, 'byteorder'): if trace.stats.mseed.byteorder in [0, 1, -1]: trace_attr['byteorder'] = trace.stats.mseed.byteorder elif trace.stats.mseed.byteorder == '=': if NATIVE_BYTEORDER == '<': trace_attr['byteorder'] = 0 else: trace_attr['byteorder'] = 1 elif trace.stats.mseed.byteorder == '<': trace_attr['byteorder'] = 0 elif trace.stats.mseed.byteorder == '>': trace_attr['byteorder'] = 1 else: msg = "Invalid byteorder in Stream[%i].stats." % _i + \ "mseed.byteorder. It must be either '<', '>', '='," + \ " 0, 1 or -1" raise ValueError(msg) else: trace_attr['byteorder'] = 1 if trace_attr['byteorder'] == -1: if NATIVE_BYTEORDER == '<': trace_attr['byteorder'] = 0 else: trace_attr['byteorder'] = 1 # Handle the encoding. trace_attr['encoding'] = None # If encoding arrives here it is already guaranteed to be a valid # integer encoding. if encoding is not None: # Check if the dtype for all traces is compatible with the enforced # encoding. ident, _, dtype, _ = ENCODINGS[encoding] if trace.data.dtype.type != dtype: msg = """ Wrong dtype for Stream[%i].data for encoding %s. Please change the dtype of your data or use an appropriate encoding. See the obspy.io.mseed documentation for more information. """ % (_i, ident) raise Exception(msg) trace_attr['encoding'] = encoding elif hasattr(trace.stats, 'mseed') and hasattr(trace.stats.mseed, 'encoding'): trace_attr["encoding"] = \ util._convert_and_check_encoding_for_writing( trace.stats.mseed.encoding) # Check if the encoding matches the data's dtype. if trace.data.dtype.type != ENCODINGS[trace_attr['encoding']][2]: msg = 'The encoding specified in ' + \ 'trace.stats.mseed.encoding does not match the ' + \ 'dtype of the data.\nA suitable encoding will ' + \ 'be chosen.' warnings.warn(msg, UserWarning) trace_attr['encoding'] = None # automatically detect encoding if no encoding is given. if not trace_attr['encoding']: if trace.data.dtype.type == np.int32: trace_attr['encoding'] = 11 elif trace.data.dtype.type == np.float32: trace_attr['encoding'] = 4 elif trace.data.dtype.type == np.float64: trace_attr['encoding'] = 5 elif trace.data.dtype.type == np.int16: trace_attr['encoding'] = 1 elif trace.data.dtype.type == np.dtype(native_str('|S1')).type: trace_attr['encoding'] = 0 else: msg = "Unsupported data type %s in Stream[%i].data" % \ (trace.data.dtype, _i) raise Exception(msg) # Convert data if necessary, otherwise store references in list. if trace_attr['encoding'] == 1: # INT16 needs INT32 data type trace_data.append(trace.data.copy().astype(np.int32)) else: trace_data.append(trace.data) # Do some final sanity checks and raise a warning if a file will be written # with more than one different encoding, record length or byte order. encodings = {_i['encoding'] for _i in trace_attributes} reclens = {_i['reclen'] for _i in trace_attributes} byteorders = {_i['byteorder'] for _i in trace_attributes} msg = 'File will be written with more than one different %s.\n' + \ 'This might have a negative influence on the compatibility ' + \ 'with other programs.' if len(encodings) != 1: warnings.warn(msg % 'encodings') if len(reclens) != 1: warnings.warn(msg % 'record lengths') if len(byteorders) != 1: warnings.warn(msg % 'byteorders') # Open filehandler or use an existing file like object. if not hasattr(filename, 'write'): f = open(filename, 'wb') else: f = filename # Loop over every trace and finally write it to the filehandler. for trace, data, trace_attr in zip(stream, trace_data, trace_attributes): if not len(data): msg = 'Skipping empty trace "%s".' % (trace) warnings.warn(msg) continue # Create C struct MSTrace. mst = MST(trace, data, dataquality=trace_attr['dataquality']) # Initialize packedsamples pointer for the mst_pack function packedsamples = C.c_int() # Callback function for mst_pack to actually write the file def record_handler(record, reclen, _stream): f.write(record[0:reclen]) # Define Python callback function for use in C function rec_handler = C.CFUNCTYPE(C.c_void_p, C.POINTER(C.c_char), C.c_int, C.c_void_p)(record_handler) # Fill up msr record structure, this is already contained in # mstg, however if blk1001 is set we need it anyway msr = clibmseed.msr_init(None) msr.contents.network = trace.stats.network.encode('ascii', 'strict') msr.contents.station = trace.stats.station.encode('ascii', 'strict') msr.contents.location = trace.stats.location.encode('ascii', 'strict') msr.contents.channel = trace.stats.channel.encode('ascii', 'strict') msr.contents.dataquality = trace_attr['dataquality'].\ encode('ascii', 'strict') # Set starting sequence number msr.contents.sequence_number = trace_attr['sequence_number'] # Only use Blockette 1001 if necessary. if use_blkt_1001: # Timing quality has been set in trace_attr size = C.sizeof(Blkt1001S) # Only timing quality matters here, other blockette attributes will # be filled by libmseed.msr_normalize_header blkt_value = pack(native_str("BBBB"), trace_attr['timing_quality'], 0, 0, 0) blkt_ptr = C.create_string_buffer(blkt_value, len(blkt_value)) # Usually returns a pointer to the added blockette in the # blockette link chain and a NULL pointer if it fails. # NULL pointers have a false boolean value according to the # ctypes manual. ret_val = clibmseed.msr_addblockette(msr, blkt_ptr, size, 1001, 0) if bool(ret_val) is False: clibmseed.msr_free(C.pointer(msr)) del msr raise Exception('Error in msr_addblockette') # Only use Blockette 100 if necessary. # Determine if a blockette 100 will be needed to represent the input # sample rate or if the sample rate in the fixed section of the data # header will suffice (see ms_genfactmult in libmseed/genutils.c) use_blkt_100 = False _factor = C.c_int16() _multiplier = C.c_int16() _retval = clibmseed.ms_genfactmult(trace.stats.sampling_rate, C.pointer(_factor), C.pointer(_multiplier)) # Use blockette 100 if ms_genfactmult() failed. if _retval != 0: use_blkt_100 = True # Otherwise figure out if ms_genfactmult() found exact factors. # Otherwise write blockette 100. else: ms_sr = clibmseed.ms_nomsamprate(_factor.value, _multiplier.value) # It is also necessary if the libmseed calculated sampling rate # would result in a loss of accuracy - the floating point # comparision is on purpose here as it will always try to # preserve all accuracy. # Cast to float32 to not add blockette 100 for values # that cannot be represented with 32bits. if np.float32(ms_sr) != np.float32(trace.stats.sampling_rate): use_blkt_100 = True if use_blkt_100: size = C.sizeof(Blkt100S) blkt100 = C.c_char(b' ') C.memset(C.pointer(blkt100), 0, size) ret_val = clibmseed.msr_addblockette(msr, C.pointer(blkt100), size, 100, 0) # NOQA # Usually returns a pointer to the added blockette in the # blockette link chain and a NULL pointer if it fails. # NULL pointers have a false boolean value according to the # ctypes manual. if bool(ret_val) is False: clibmseed.msr_free(C.pointer(msr)) # NOQA del msr # NOQA raise Exception('Error in msr_addblockette') # Pack mstg into a MSEED file using the callback record_handler as # write method. errcode = clibmseed.mst_pack(mst.mst, rec_handler, None, trace_attr['reclen'], trace_attr['encoding'], trace_attr['byteorder'], C.byref(packedsamples), flush, verbose, msr) # NOQA if errcode == 0: msg = ("Did not write any data for trace '%s' even though it " "contains data values.") % trace raise ValueError(msg) if errcode == -1: clibmseed.msr_free(C.pointer(msr)) # NOQA del mst, msr # NOQA raise Exception('Error in mst_pack') # Deallocate any allocated memory. clibmseed.msr_free(C.pointer(msr)) # NOQA del mst, msr # NOQA # Close if its a file handler. if not hasattr(filename, 'write'): f.close()
def isSAC(filename): """ Checks whether a file is a SAC file or not. :type filename: str :param filename: SAC file to be checked. :rtype: bool :return: ``True`` if a SAC file. .. rubric:: Example >>> isSAC('/path/to/test.sac') #doctest: +SKIP """ try: with open(filename, 'rb') as f: # read delta (first header float) delta_bin = f.read(4) delta = struct.unpack(native_str('<f'), delta_bin)[0] # read nvhdr (70 header floats, 6 position in header integers) f.seek(4 * 70 + 4 * 6) nvhdr_bin = f.read(4) nvhdr = struct.unpack(native_str('<i'), nvhdr_bin)[0] # read leven (70 header floats, 35 header integers, 0 position in # header bool) f.seek(4 * 70 + 4 * 35) leven_bin = f.read(4) leven = struct.unpack(native_str('<i'), leven_bin)[0] # read lpspol (70 header floats, 35 header integers, 1 position in # header bool) f.seek(4 * 70 + 4 * 35 + 4 * 1) lpspol_bin = f.read(4) lpspol = struct.unpack(native_str('<i'), lpspol_bin)[0] # read lovrok (70 header floats, 35 header integers, 2 position in # header bool) f.seek(4 * 70 + 4 * 35 + 4 * 2) lovrok_bin = f.read(4) lovrok = struct.unpack(native_str('<i'), lovrok_bin)[0] # read lcalda (70 header floats, 35 header integers, 3 position in # header bool) f.seek(4 * 70 + 4 * 35 + 4 * 3) lcalda_bin = f.read(4) lcalda = struct.unpack(native_str('<i'), lcalda_bin)[0] # check if file is big-endian if nvhdr < 0 or nvhdr > 20: nvhdr = struct.unpack(native_str('>i'), nvhdr_bin)[0] delta = struct.unpack(native_str('>f'), delta_bin)[0] leven = struct.unpack(native_str('>i'), leven_bin)[0] lpspol = struct.unpack(native_str('>i'), lpspol_bin)[0] lovrok = struct.unpack(native_str('>i'), lovrok_bin)[0] lcalda = struct.unpack(native_str('>i'), lcalda_bin)[0] # check again nvhdr if nvhdr < 1 or nvhdr > 20: return False if delta <= 0: return False if leven != 0 and leven != 1 and leven != -12345: return False if lpspol != 0 and lpspol != 1 and lpspol != -12345: return False if lovrok != 0 and lovrok != 1 and lovrok != -12345: return False if lcalda != 0 and lcalda != 1 and lcalda != -12345: return False except: return False return True
def test_ppsd_w_iris(self): # Bands to be used this is the upper and lower frequency band pairs fres = zip([0.1, 0.05], [0.2, 0.1]) file_data_anmo = os.path.join(self.path, 'IUANMO.seed') # Read in ANMO data for one day st = read(file_data_anmo) # Use a canned ANMO response which will stay static paz = { 'gain': 86298.5, 'zeros': [0, 0], 'poles': [ -59.4313, -22.7121 + 27.1065j, -22.7121 + 27.1065j, -0.0048004, -0.073199 ], 'sensitivity': 3.3554 * 10**9 } # Make an empty PPSD and add the data # use highest frequency given by IRIS Mustang noise-pdf web service # (0.475683 Hz == 2.10224036 s) as center of first bin, so that we # end up with the same bins. ppsd = PPSD(st[0].stats, paz, period_limits=(2.10224036, 1400)) ppsd.add(st) ppsd.calculate_histogram() # Get the 50th percentile from the PPSD (per, perval) = ppsd.get_percentile(percentile=50) perinv = 1 / per # Read in the results obtained from a Mustang flat file file_data_iris = os.path.join(self.path, 'IRISpdfExample') data = np.genfromtxt(file_data_iris, comments='#', delimiter=',', dtype=[(native_str("freq"), np.float64), (native_str("power"), np.int32), (native_str("hits"), np.int32)]) freq = data["freq"] power = data["power"] hits = data["hits"] # cut data to same period range as in the ppsd we computed # (Mustang returns more long periods, probably due to some zero padding # or longer nfft in psd) num_periods = len(ppsd.period_bin_centers) freqdistinct = np.array(sorted(set(freq), reverse=True)[:num_periods]) # just make sure that we compare the same periods in the following # (as we access both frequency arrays by indices from now on) np.testing.assert_allclose(freqdistinct, 1 / ppsd.period_bin_centers, rtol=1e-4) # For each frequency pair we want to compare the mean of the bands for fre in fres: # determine which bins we want to compare mask = (fre[0] < perinv) & (perinv < fre[1]) # Get the values for the bands from the PPSD per_val_good_obspy = perval[mask] percenlist = [] # Now we sort out all of the data from the IRIS flat file # We will loop through the frequency values and compute a # 50th percentile for curfreq in freqdistinct[mask]: mask_ = curfreq == freq tempvalslist = np.repeat(power[mask_], hits[mask_]) percenlist.append(np.percentile(tempvalslist, 50)) # Here is the actual test np.testing.assert_allclose(np.mean(per_val_good_obspy), np.mean(percenlist), rtol=0.0, atol=1.2)
def twisted(self, flask_app): from twisted.internet import reactor from twisted.internet.error import ReactorNotRunning from twisted.web import server from twisted.web.wsgi import WSGIResource from twisted.application import service, strports from twisted.scripts._twistd_unix import ServerOptions from twisted.scripts._twistd_unix import UnixApplicationRunner if self.conf['secure']: self.conf['strport'] = 'ssl:{port}:'\ 'interface={interface}:'\ 'privateKey={priv_pem}:'\ 'certKey={pub_pem}'.format(**self.conf) else: self.conf['strport'] = 'tcp:{port}:'\ 'interface={interface}'.format(**self.conf) # Options as in twistd command line utility self.conf['twisted_opts'] = '--pidfile={pidfile} -no'.format( **self.conf).split() #################################################################### # See # http://twistedmatrix.com/documents/current/web/howto/ # using-twistedweb.html # (Serving WSGI Applications) for the basic ideas of the below code #################################################################### def my_sigint(x, n): try: reactor.stop() except ReactorNotRunning: pass signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGINT, my_sigint) resource = WSGIResource(reactor, reactor.getThreadPool(), flask_app) class QuietSite(server.Site): def log(*args, **kwargs): '''Override the logging so that requests are not logged''' pass # Log only errors, not every page hit site = QuietSite(resource) # To log every single page hit, uncomment the following line # site = server.Site(resource) application = service.Application("Sage Notebook") s = strports.service(native_str(self.conf['strport']), site) self.open_page() s.setServiceParent(application) # This has to be done after sagenb.create_app is run reactor.addSystemEventTrigger('before', 'shutdown', self.save_notebook) # Run the application without .tac file class AppRunner(UnixApplicationRunner): ''' twisted application runner. The application is provided on init, not read from file ''' def __init__(self, app, conf): super(self.__class__, self).__init__(conf) self.app = app def createOrGetApplication(self): '''Overrides the reading of the application from file''' return self.app twisted_conf = ServerOptions() twisted_conf.parseOptions(self.conf['twisted_opts']) AppRunner(application, twisted_conf).run()
def _event_type_class_factory(class_name, class_attributes=[], class_contains=[]): """ Class factory to unify the creation of all the types needed for the event handling in ObsPy. The types oftentimes share attributes and setting them manually every time is cumbersome, error-prone and hard to do consistently. The classes created with this method will inherit from :class:`~obspy.core.util.AttribDict`. Usage to create a new class type: The created class will assure that any given (key, type) attribute pairs will always be of the given type and will attempt to convert any given value to the correct type and raise an error otherwise. This happens to values given during initialization as well as values set when the object has already been created. A useful type is Enum if you want to restrict the acceptable values. >>> from obspy.core.util import Enum >>> MyEnum = Enum(["a", "b", "c"]) >>> class_attributes = [ \ ("resource_id", ResourceIdentifier), \ ("creation_info", CreationInfo), \ ("some_letters", MyEnum), \ ("some_error_quantity", float, ATTRIBUTE_HAS_ERRORS), \ ("description", str)] Furthermore the class can contain lists of other objects. There is not much to it so far. Giving the name of the created class is mandatory. >>> class_contains = ["comments"] >>> TestEventClass = _event_type_class_factory("TestEventClass", \ class_attributes=class_attributes, \ class_contains=class_contains) >>> assert(TestEventClass.__name__ == "TestEventClass") Now the new class type can be used. >>> test_event = TestEventClass(resource_id="event/123456", \ creation_info={"author": "obspy.org", "version": "0.1"}) All given arguments will be converted to the right type upon setting them. >>> test_event.resource_id ResourceIdentifier(id="event/123456") >>> print(test_event.creation_info) CreationInfo(author='obspy.org', version='0.1') All others will be set to None. >>> assert(test_event.description is None) >>> assert(test_event.some_letters is None) If the resource_id attribute of the created class type is set, the object the ResourceIdentifier refers to will be the class instance. >>> assert(id(test_event) == \ id(test_event.resource_id.get_referred_object())) They can be set later and will be converted to the appropriate type if possible. >>> test_event.description = 1 >>> assert(test_event.description == "1") Trying to set with an inappropriate value will raise an error. >>> test_event.some_letters = "d" # doctest:+ELLIPSIS Traceback (most recent call last): ... ValueError: Setting attribute "some_letters" failed. ... If you pass ``ATTRIBUTE_HAS_ERRORS`` as the third tuple item for the class_attributes, a error (type :class:`~obspy.core.event.base.QuantityError`) will be be created that will be named like the attribute with "_errors" appended. >>> assert(hasattr(test_event, "some_error_quantity_errors")) >>> test_event.some_error_quantity_errors # doctest: +ELLIPSIS QuantityError(...) """ class AbstractEventType(AttribDict): # Keep the class attributes in a class level list for a manual property # implementation that works when inheriting from AttribDict. _properties = [] for item in class_attributes: _properties.append((item[0], item[1])) if len(item) == 3 and item[2] == ATTRIBUTE_HAS_ERRORS: _properties.append((item[0] + "_errors", QuantityError)) _property_keys = [_i[0] for _i in _properties] _property_dict = {} for key, value in _properties: _property_dict[key] = value _containers = class_contains warn_on_non_default_key = True defaults = dict.fromkeys(class_contains, []) defaults.update(dict.fromkeys(_property_keys, None)) do_not_warn_on = ["extra"] def __init__(self, *args, **kwargs): # Make sure the args work as expected. Therefore any specified # arg will overwrite a potential kwarg, e.g. arg at position 0 will # overwrite kwargs class_attributes[0]. for _i, item in enumerate(args): # Use the class_attributes list here because it is not yet # polluted be the error quantities. kwargs[class_attributes[_i][0]] = item # Set all property values to None or the kwarg value. for key, _ in self._properties: value = kwargs.get(key, None) # special handling for resource id if key == "resource_id": if kwargs.get("force_resource_id", False): if value is None: value = ResourceIdentifier() setattr(self, key, value) # Containers currently are simple lists. for name in self._containers: setattr(self, name, list(kwargs.get(name, []))) # All errors are QuantityError. If they are not set yet, set them # now. for key, _ in self._properties: if key.endswith("_errors") and getattr(self, key) is None: setattr(self, key, QuantityError()) def clear(self): super(AbstractEventType, self).clear() self.__init__(force_resource_id=False) def __str__(self, force_one_line=False): """ Fairly extensive in an attempt to cover several use cases. It is always possible to change it in the child class. """ # Get the attribute and containers that are to be printed. Only not # None attributes and non-error attributes are printed. The errors # will appear behind the actual value. # We use custom _bool() for testing getattr() since we want to # print int and float values that are equal to zero and empty # strings. attributes = [_i for _i in self._property_keys if not _i.endswith("_errors") and _bool(getattr(self, _i))] containers = [_i for _i in self._containers if _bool(getattr(self, _i))] # Get the longest attribute/container name to print all of them # nicely aligned. max_length = max(max([len(_i) for _i in attributes]) if attributes else 0, max([len(_i) for _i in containers]) if containers else 0) + 1 ret_str = self.__class__.__name__ # Case 1: Empty object. if not attributes and not containers: return ret_str + "()" def get_value_repr(key): value = getattr(self, key) if isinstance(value, (str, native_str)): value = native_str(value) repr_str = value.__repr__() # Print any associated errors. error_key = key + "_errors" if self.get(error_key, False): err_items = sorted(getattr(self, error_key).items()) repr_str += " [%s]" % ', '.join( sorted([str(k) + "=" + str(v) for k, v in err_items if v is not None])) return repr_str # Case 2: Short representation for small objects. Will just print a # single line. if len(attributes) <= 3 and not containers or\ force_one_line: att_strs = ["%s=%s" % (_i, get_value_repr(_i)) for _i in attributes if _bool(getattr(self, _i))] ret_str += "(%s)" % ", ".join(att_strs) return ret_str # Case 3: Verbose string representation for large object. if attributes: format_str = "%" + str(max_length) + "s: %s" att_strs = [format_str % (_i, get_value_repr(_i)) for _i in attributes if _bool(getattr(self, _i))] ret_str += "\n\t" + "\n\t".join(att_strs) # For the containers just print the number of elements in each. if containers: # Print delimiter only if there are attributes. if attributes: ret_str += '\n\t' + '---------'.rjust(max_length + 5) element_str = "%" + str(max_length) + "s: %i Elements" ret_str += "\n\t" + \ "\n\t".join( [element_str % (_i, len(getattr(self, _i))) for _i in containers]) return ret_str def _repr_pretty_(self, p, cycle): p.text(str(self)) def copy(self): return copy.deepcopy(self) def __repr__(self): return self.__str__(force_one_line=True) # called for bool on PY2 def __nonzero__(self): return self.__bool__() def __bool__(self): # We use custom _bool() for testing getattr() since we want # zero valued int and float and empty string attributes to be True. if any([_bool(getattr(self, _i)) for _i in self._property_keys + self._containers]): return True return False def __eq__(self, other): """ Two instances are considered equal if all attributes and all lists are identical. """ # Looping should be quicker on average than a list comprehension # because only the first non-equal attribute will already return. for attrib in self._property_keys: if not hasattr(other, attrib) or \ (getattr(self, attrib) != getattr(other, attrib)): return False for container in self._containers: if not hasattr(other, container) or \ (getattr(self, container) != getattr(other, container)): return False return True def __ne__(self, other): return not self.__eq__(other) def __setattr__(self, name, value): """ Custom property implementation that works if the class is inheriting from AttribDict. """ # Pass to the parent method if not a custom property. if name not in self._property_dict.keys(): AttribDict.__setattr__(self, name, value) return attrib_type = self._property_dict[name] # If the value is None or already the correct type just set it. if (value is not None) and (type(value) is not attrib_type): # If it is a dict, and the attrib_type is no dict, than all # values will be assumed to be keyword arguments. if isinstance(value, dict): new_value = attrib_type(**value) else: new_value = attrib_type(value) if new_value is None: msg = 'Setting attribute "%s" failed. ' % (name) msg += 'Value "%s" could not be converted to type "%s"' % \ (str(value), str(attrib_type)) raise ValueError(msg) value = new_value AttribDict.__setattr__(self, name, value) # If "name" is resource_id and value is not None, set the referred # object of the ResourceIdentifier to self. if name == "resource_id" and value is not None: self.resource_id.set_referred_object(self) class AbstractEventTypeWithResourceID(AbstractEventType): def __init__(self, force_resource_id=True, *args, **kwargs): kwargs["force_resource_id"] = force_resource_id super(AbstractEventTypeWithResourceID, self).__init__(*args, **kwargs) if "resource_id" in [item[0] for item in class_attributes]: base_class = AbstractEventTypeWithResourceID else: base_class = AbstractEventType # Set the class type name. setattr(base_class, "__name__", native_str(class_name)) return base_class
def _create_report(ttrs, timetaken, log, server, hostname, sorted_tests, ci_url=None, pr_url=None, import_failures=None): # import additional libraries here to speed up normal tests from future import standard_library with standard_library.hooks(): import urllib.parse import http.client import codecs from xml.etree import ElementTree from xml.sax.saxutils import escape if import_failures is None: import_failures = {} timestamp = int(time.time()) result = {'timestamp': timestamp} result['slowest_tests'] = [("%0.3fs" % dt, "%s" % desc) for (desc, dt) in sorted_tests[:20]] result['timetaken'] = timetaken if log: try: data = codecs.open(log, 'r', encoding='UTF-8').read() result['install_log'] = escape(data) except Exception: print("Cannot open log file %s" % log) # get ObsPy module versions result['obspy'] = {} tests = 0 errors = 0 failures = 0 skipped = 0 try: installed = get_git_version() except Exception: installed = '' result['obspy']['installed'] = installed for module in sorted(ALL_MODULES): result['obspy'][module] = {} result['obspy'][module]['installed'] = installed # add a failed-to-import test module to report with an error if module in import_failures: result['obspy'][module]['timetaken'] = 0 result['obspy'][module]['tested'] = True result['obspy'][module]['tests'] = 1 # can't say how many tests would have been in that suite so just # leave 0 result['obspy'][module]['skipped'] = 0 result['obspy'][module]['failures'] = {} result['obspy'][module]['errors'] = { 'f%s' % (errors): import_failures[module] } tests += 1 errors += 1 continue if module not in ttrs: continue # test results ttr = ttrs[module] result['obspy'][module]['timetaken'] = ttr.__dict__['timetaken'] result['obspy'][module]['tested'] = True result['obspy'][module]['tests'] = ttr.testsRun skipped += len(ttr.skipped) result['obspy'][module]['skipped'] = len(ttr.skipped) tests += ttr.testsRun # depending on module type either use failure (network related modules) # or errors (all others) result['obspy'][module]['errors'] = {} result['obspy'][module]['failures'] = {} if module in NETWORK_MODULES: for _, text in ttr.errors: result['obspy'][module]['failures']['f%s' % (failures)] = text failures += 1 for _, text in ttr.failures: result['obspy'][module]['failures']['f%s' % (failures)] = text failures += 1 else: for _, text in ttr.errors: result['obspy'][module]['errors']['f%s' % (errors)] = text errors += 1 for _, text in ttr.failures: result['obspy'][module]['errors']['f%s' % (errors)] = text errors += 1 # get dependencies result['dependencies'] = {} for module in DEPENDENCIES: if module == "pep8-naming": module_ = "pep8ext_naming" else: module_ = module temp = module_.split('.') try: mod = __import__(module_, fromlist=[native_str(temp[1:])]) except ImportError: version_ = '---' else: try: version_ = mod.__version__ except AttributeError: version_ = '???' result['dependencies'][module] = version_ # get system / environment settings result['platform'] = {} for func in [ 'system', 'release', 'version', 'machine', 'processor', 'python_version', 'python_implementation', 'python_compiler', 'architecture' ]: try: temp = getattr(platform, func)() if isinstance(temp, tuple): temp = temp[0] result['platform'][func] = temp except Exception: result['platform'][func] = '' # set node name to hostname if set result['platform']['node'] = hostname # post only the first part of the node name (only applies to MacOS X) try: result['platform']['node'] = result['platform']['node'].split('.')[0] except Exception: pass # test results result['tests'] = tests result['errors'] = errors result['failures'] = failures result['skipped'] = skipped # try to append info on skipped tests: result['skipped_tests_details'] = [] try: for module, testresult_ in ttrs.items(): if testresult_.skipped: for skipped_test, skip_message in testresult_.skipped: result['skipped_tests_details'].append( (module, skipped_test.__module__, skipped_test.__class__.__name__, skipped_test._testMethodName, skip_message)) except Exception: exc_type, exc_value, exc_tb = sys.exc_info() print("\n".join(traceback.format_exception(exc_type, exc_value, exc_tb))) result['skipped_tests_details'] = [] if ci_url is not None: result['ciurl'] = ci_url if pr_url is not None: result['prurl'] = pr_url # generate XML document def _dict2xml(doc, result): for key, value in result.items(): key = key.split('(')[0].strip() if isinstance(value, dict): child = ElementTree.SubElement(doc, key) _dict2xml(child, value) elif value is not None: if isinstance(value, (str, native_str)): ElementTree.SubElement(doc, key).text = value elif isinstance(value, (str, native_str)): ElementTree.SubElement(doc, key).text = str(value, 'utf-8') else: ElementTree.SubElement(doc, key).text = str(value) else: ElementTree.SubElement(doc, key) root = ElementTree.Element("report") _dict2xml(root, result) xml_doc = ElementTree.tostring(root) print() # send result to report server params = urllib.parse.urlencode({ 'timestamp': timestamp, 'system': result['platform']['system'], 'python_version': result['platform']['python_version'], 'architecture': result['platform']['architecture'], 'tests': tests, 'failures': failures, 'errors': errors, 'modules': len(ttrs) + len(import_failures), 'xml': xml_doc }) headers = { "Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain" } conn = http.client.HTTPConnection(server) conn.request("POST", "/", params, headers) # get the response response = conn.getresponse() # handle redirect if response.status == 301: o = urllib.parse.urlparse(response.msg['location']) conn = http.client.HTTPConnection(o.netloc) conn.request("POST", o.path, params, headers) # get the response response = conn.getresponse() # handle errors if response.status == 200: print("Test report has been sent to %s. Thank you!" % (server)) else: print("Error: Could not sent a test report to %s." % (server)) print(response.reason) conn.close()
def _internal_is_sac(buf): """ Checks whether a file-like object contains a SAC file or not. :param buf: SAC file to be checked. :type buf: file-like object or open file. :rtype: bool :return: ``True`` if a SAC file. """ starting_pos = buf.tell() try: # read delta (first header float) delta_bin = buf.read(4) delta = struct.unpack(native_str('<f'), delta_bin)[0] # read nvhdr (70 header floats, 6 position in header integers) buf.seek(starting_pos + 4 * 70 + 4 * 6, 0) nvhdr_bin = buf.read(4) nvhdr = struct.unpack(native_str('<i'), nvhdr_bin)[0] # read leven (70 header floats, 35 header integers, 0 position in # header bool) buf.seek(starting_pos + 4 * 70 + 4 * 35, 0) leven_bin = buf.read(4) leven = struct.unpack(native_str('<i'), leven_bin)[0] # read lpspol (70 header floats, 35 header integers, 1 position in # header bool) buf.seek(starting_pos + 4 * 70 + 4 * 35 + 4 * 1, 0) lpspol_bin = buf.read(4) lpspol = struct.unpack(native_str('<i'), lpspol_bin)[0] # read lovrok (70 header floats, 35 header integers, 2 position in # header bool) buf.seek(starting_pos + 4 * 70 + 4 * 35 + 4 * 2, 0) lovrok_bin = buf.read(4) lovrok = struct.unpack(native_str('<i'), lovrok_bin)[0] # read lcalda (70 header floats, 35 header integers, 3 position in # header bool) buf.seek(starting_pos + 4 * 70 + 4 * 35 + 4 * 3, 0) lcalda_bin = buf.read(4) lcalda = struct.unpack(native_str('<i'), lcalda_bin)[0] # check if file is big-endian if nvhdr < 0 or nvhdr > 20: nvhdr = struct.unpack(native_str('>i'), nvhdr_bin)[0] delta = struct.unpack(native_str('>f'), delta_bin)[0] leven = struct.unpack(native_str('>i'), leven_bin)[0] lpspol = struct.unpack(native_str('>i'), lpspol_bin)[0] lovrok = struct.unpack(native_str('>i'), lovrok_bin)[0] lcalda = struct.unpack(native_str('>i'), lcalda_bin)[0] # check again nvhdr if nvhdr < 1 or nvhdr > 20: return False if delta <= 0: return False if leven != 0 and leven != 1 and leven != -12345: return False if lpspol != 0 and lpspol != 1 and lpspol != -12345: return False if lovrok != 0 and lovrok != 1 and lovrok != -12345: return False if lcalda != 0 and lcalda != 1 and lcalda != -12345: return False except Exception: return False finally: # Reset buffer head position after reading. buf.seek(starting_pos, 0) return True
def png_pack(png_tag, data): chunk_head = png_tag + data return ( struct.pack(native_str("!I"), len(data)) + chunk_head + struct.pack(native_str("!I"), 0xFFFFFFFF & zlib.crc32(chunk_head)))
def _write_q(stream, filename, data_directory=None, byteorder='=', append=False, **kwargs): # @UnusedVariable """ Writes a Seismic Handler Q file from given ObsPy Stream object. .. warning:: This function should NOT be called directly, it registers via the the :meth:`~obspy.core.stream.Stream.write` method of an ObsPy :class:`~obspy.core.stream.Stream` object, call this instead. :type stream: :class:`~obspy.core.stream.Stream` :param stream: The ObsPy Stream object to write. :type filename: str :param filename: Name of the Q file to write. :type data_directory: str, optional :param data_directory: Data directory where the corresponding QBN will be written. :type byteorder: str, optional :param byteorder: Enforce byte order for data file. Can be little endian (``'<'``), big endian (``'>'``), or native byte order (``'='``). Defaults to ``'='``. :type append: bool, optional :param append: If filename exists append all data to file, default False. """ if filename.endswith('.QHD') or filename.endswith('.QBN'): filename = os.path.splitext(filename)[0] if data_directory: temp = os.path.basename(filename) filename_data = os.path.join(data_directory, temp) else: filename_data = filename filename_header = filename + '.QHD' # if the header file exists its assumed that the data is also there if os.path.exists(filename_header) and append: try: trcs = _read_q(filename_header, headonly=True) mode = 'ab' count_offset = len(trcs) except: raise Exception("Target filename '%s' not readable!" % filename) else: append = False mode = 'wb' count_offset = 0 fh = open(filename_header, mode) fh_data = open(filename_data + '.QBN', mode) # build up header strings headers = [] minnol = 4 cur_npts = 0 for trace in stream: temp = "L000:%d~ " % cur_npts cur_npts += trace.stats.npts temp += "L001:%d~ R000:%f~ R026:%f~ " % (trace.stats.npts, trace.stats.delta, trace.stats.calib) if trace.stats.station: temp += "S001:%s~ " % trace.stats.station # component must be split if len(trace.stats.channel) > 2: temp += "C000:%c~ " % trace.stats.channel[2] if len(trace.stats.channel) > 0: temp += "C001:%c~ " % trace.stats.channel[0] if len(trace.stats.channel) > 1: temp += "C002:%c~ " % trace.stats.channel[1] # special format for start time dt = trace.stats.starttime temp += "S021:%s~ " % from_utcdatetime(dt) for key, value in trace.stats.get('sh', {}).items(): # skip unknown keys if not key or key not in SH_IDX.keys(): continue # convert UTCDateTimes into strings if isinstance(value, UTCDateTime): value = from_utcdatetime(value) temp += "%s:%s~ " % (SH_IDX[key], value) headers.append(temp) # get maximal number of trclines nol = len(temp) // 74 + 1 if nol > minnol: minnol = nol # first line: magic number, cmtlines, trclines # XXX: comment lines are ignored if not append: line = "43981 1 %d\n" % minnol fh.write(line.encode('ascii', 'strict')) for i, trace in enumerate(stream): # write headers temp = [headers[i][j:j + 74] for j in range(0, len(headers[i]), 74)] for j in range(0, minnol): try: line = "%02d|%s\n" % ((i + 1 + count_offset) % 100, temp[j]) fh.write(line.encode('ascii', 'strict')) except: line = "%02d|\n" % ((i + 1 + count_offset) % 100) fh.write(line.encode('ascii', 'strict')) # write data in given byte order dtype = native_str(byteorder + 'f4') data = np.require(trace.data, dtype=dtype) fh_data.write(data.data) fh.close() fh_data.close()
def start(self, info_hash): return self._server.d.start(native_str(info_hash))
SEED_CONTROL_HEADERS = [ord('V'), ord('A'), ord('S'), ord('T')] MINI_SEED_CONTROL_HEADERS = [ord('D'), ord('R'), ord('Q'), ord('M')] VALID_CONTROL_HEADERS = SEED_CONTROL_HEADERS + MINI_SEED_CONTROL_HEADERS # expected data types for libmseed id: (numpy, ctypes) DATATYPES = {b"a": C.c_char, b"i": C.c_int32, b"f": C.c_float, b"d": C.c_double} SAMPLESIZES = {'a': 1, 'i': 4, 'f': 4, 'd': 8} # Valid record lengths for Mini-SEED files. VALID_RECORD_LENGTHS = [256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576] # allowed encodings: # id: (name, sampletype a/i/f/d, default NumPy type, write support) ENCODINGS = {0: ("ASCII", "a", np.dtype(native_str("|S1")).type, True), 1: ("INT16", "i", np.dtype(np.int16), True), 3: ("INT32", "i", np.dtype(np.int32), True), 4: ("FLOAT32", "f", np.dtype(np.float32), True), 5: ("FLOAT64", "d", np.dtype(np.float64), True), 10: ("STEIM1", "i", np.dtype(np.int32), True), 11: ("STEIM2", "i", np.dtype(np.int32), True), 12: ("GEOSCOPE24", "f", np.dtype(np.float32), False), 13: ("GEOSCOPE16_3", "f", np.dtype(np.float32), False), 14: ("GEOSCOPE16_4", "f", np.dtype(np.float32), False), 16: ("CDSN", "i", np.dtype(np.int32), False), 30: ("SRO", "i", np.dtype(np.int32), False), 32: ("DWWSSN", "i", np.dtype(np.int32), False)} # Encodings not supported by libmseed and consequently ObsPy. UNSUPPORTED_ENCODINGS = {
def _createReport(ttrs, timetaken, log, server, hostname, sorted_tests): # import additional libraries here to speed up normal tests from future import standard_library with standard_library.hooks(): import urllib.parse import http.client from xml.sax.saxutils import escape import codecs from xml.etree import ElementTree as etree timestamp = int(time.time()) result = {'timestamp': timestamp} result['slowest_tests'] = [("%0.3fs" % dt, "%s" % desc) for (desc, dt) in sorted_tests[:20]] result['timetaken'] = timetaken if log: try: data = codecs.open(log, 'r', encoding='UTF-8').read() result['install_log'] = escape(data) except: print("Cannot open log file %s" % log) # get ObsPy module versions result['obspy'] = {} tests = 0 errors = 0 failures = 0 skipped = 0 try: installed = get_git_version() except: installed = '' result['obspy']['installed'] = installed for module in sorted(ALL_MODULES): result['obspy'][module] = {} if module not in ttrs: continue result['obspy'][module]['installed'] = installed # test results ttr = ttrs[module] result['obspy'][module]['timetaken'] = ttr.__dict__['timetaken'] result['obspy'][module]['tested'] = True result['obspy'][module]['tests'] = ttr.testsRun # skipped is not supported for Python < 2.7 try: skipped += len(ttr.skipped) result['obspy'][module]['skipped'] = len(ttr.skipped) except AttributeError: skipped = '' result['obspy'][module]['skipped'] = '' tests += ttr.testsRun # depending on module type either use failure (network related modules) # or errors (all others) result['obspy'][module]['errors'] = {} result['obspy'][module]['failures'] = {} if module in NETWORK_MODULES: for _, text in ttr.errors: result['obspy'][module]['failures']['f%s' % (failures)] = text failures += 1 for _, text in ttr.failures: result['obspy'][module]['failures']['f%s' % (failures)] = text failures += 1 else: for _, text in ttr.errors: result['obspy'][module]['errors']['f%s' % (errors)] = text errors += 1 for _, text in ttr.failures: result['obspy'][module]['errors']['f%s' % (errors)] = text errors += 1 # get dependencies result['dependencies'] = {} for module in DEPENDENCIES: temp = module.split('.') try: mod = __import__(module, fromlist=[native_str(temp[1:])]) if module == '_omnipy': result['dependencies'][module] = mod.coreVersion() else: result['dependencies'][module] = mod.__version__ except ImportError: result['dependencies'][module] = '' # get system / environment settings result['platform'] = {} for func in ['system', 'release', 'version', 'machine', 'processor', 'python_version', 'python_implementation', 'python_compiler', 'architecture']: try: temp = getattr(platform, func)() if isinstance(temp, tuple): temp = temp[0] result['platform'][func] = temp except: result['platform'][func] = '' # set node name to hostname if set result['platform']['node'] = hostname # post only the first part of the node name (only applies to MacOS X) try: result['platform']['node'] = result['platform']['node'].split('.')[0] except: pass # test results result['tests'] = tests result['errors'] = errors result['failures'] = failures result['skipped'] = skipped # generate XML document def _dict2xml(doc, result): for key, value in result.items(): key = key.split('(')[0].strip() if isinstance(value, dict): child = etree.SubElement(doc, key) _dict2xml(child, value) elif value is not None: if isinstance(value, (str, native_str)): etree.SubElement(doc, key).text = value elif isinstance(value, (str, native_str)): etree.SubElement(doc, key).text = str(value, 'utf-8') else: etree.SubElement(doc, key).text = str(value) else: etree.SubElement(doc, key) root = etree.Element("report") _dict2xml(root, result) xml_doc = etree.tostring(root) print() # send result to report server params = urllib.parse.urlencode({ 'timestamp': timestamp, 'system': result['platform']['system'], 'python_version': result['platform']['python_version'], 'architecture': result['platform']['architecture'], 'tests': tests, 'failures': failures, 'errors': errors, 'modules': len(ttrs), 'xml': xml_doc }) headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"} conn = http.client.HTTPConnection(server) conn.request("POST", "/", params, headers) # get the response response = conn.getresponse() # handle redirect if response.status == 301: o = urllib.parse.urlparse(response.msg['location']) conn = http.client.HTTPConnection(o.netloc) conn.request("POST", o.path, params, headers) # get the response response = conn.getresponse() # handle errors if response.status == 200: print("Test report has been sent to %s. Thank you!" % (server)) else: print("Error: Could not sent a test report to %s." % (server)) print(response.reason)
def fftw_normxcorr(templates, stream, pads, threaded=False, *args, **kwargs): """ Normalised cross-correlation using the fftw library. Internally this function used double precision numbers, which is definitely required for seismic data. Cross-correlations are computed as the inverse fft of the dot product of the ffts of the stream and the reversed, normalised, templates. The cross-correlation is then normalised using the running mean and standard deviation (not using the N-1 correction) of the stream and the sums of the normalised templates. This python fucntion wraps the C-library written by C. Chamberlain for this purpose. :param templates: 2D Array of templates :type templates: np.ndarray :param stream: 1D array of continuous data :type stream: np.ndarray :param pads: List of ints of pad lengths in the same order as templates :type pads: list :param threaded: Whether to use the threaded routine or not - note openMP and python multiprocessing don't seem to play nice for this. :type threaded: bool :return: np.ndarray of cross-correlations :return: np.ndarray channels used """ utilslib = _load_cdll('libutils') argtypes = [ np.ctypeslib.ndpointer(dtype=np.float32, ndim=1, flags=native_str('C_CONTIGUOUS')), ctypes.c_int, ctypes.c_int, np.ctypeslib.ndpointer(dtype=np.float32, ndim=1, flags=native_str('C_CONTIGUOUS')), ctypes.c_int, np.ctypeslib.ndpointer(dtype=np.float32, flags=native_str('C_CONTIGUOUS')), ctypes.c_int, np.ctypeslib.ndpointer(dtype=np.intc, flags=native_str('C_CONTIGUOUS')), np.ctypeslib.ndpointer(dtype=np.intc, flags=native_str('C_CONTIGUOUS'))] restype = ctypes.c_int if threaded: func = utilslib.normxcorr_fftw_threaded else: func = utilslib.normxcorr_fftw func.argtypes = argtypes func.restype = restype # Generate a template mask used_chans = ~np.isnan(templates).any(axis=1) template_length = templates.shape[1] stream_length = len(stream) n_templates = templates.shape[0] fftshape = next_fast_len(template_length + stream_length - 1) # # Normalize and flip the templates norm = ((templates - templates.mean(axis=-1, keepdims=True)) / ( templates.std(axis=-1, keepdims=True) * template_length)) norm = np.nan_to_num(norm) ccc = np.zeros((n_templates, stream_length - template_length + 1), np.float32) used_chans_np = np.ascontiguousarray(used_chans, dtype=np.intc) pads_np = np.ascontiguousarray(pads, dtype=np.intc) ret = func( np.ascontiguousarray(norm.flatten(order='C'), np.float32), template_length, n_templates, np.ascontiguousarray(stream, np.float32), stream_length, np.ascontiguousarray(ccc, np.float32), fftshape, used_chans_np, pads_np) if ret != 0: print(ret) raise MemoryError() return ccc, used_chans
def fftw_multi_normxcorr(template_array, stream_array, pad_array, seed_ids): """ Use a C loop rather than a Python loop - in some cases this will be fast. :type template_array: dict :param template_array: :type stream_array: dict :param stream_array: :type pad_array: dict :param pad_array: :type seed_ids: list :param seed_ids: rtype: np.ndarray, list :return: 3D Array of cross-correlations and list of used channels. """ utilslib = _load_cdll('libutils') utilslib.multi_normxcorr_fftw.argtypes = [ np.ctypeslib.ndpointer(dtype=np.float32, flags=native_str('C_CONTIGUOUS')), ctypes.c_int, ctypes.c_int, ctypes.c_int, np.ctypeslib.ndpointer(dtype=np.float32, flags=native_str('C_CONTIGUOUS')), ctypes.c_int, np.ctypeslib.ndpointer(dtype=np.float32, flags=native_str('C_CONTIGUOUS')), ctypes.c_int, np.ctypeslib.ndpointer(dtype=np.intc, flags=native_str('C_CONTIGUOUS')), np.ctypeslib.ndpointer(dtype=np.intc, flags=native_str('C_CONTIGUOUS'))] utilslib.multi_normxcorr_fftw.restype = ctypes.c_int ''' Arguments are: templates (stacked [ch_1-t_1, ch_1-t_2, ..., ch_2-t_1, ch_2-t_2, ...]) number of templates template length number of channels image (stacked [ch_1, ch_2, ..., ch_n]) image length cross-correlations (stacked as per image) fft-length used channels (stacked as per templates) pad array (stacked as per templates) ''' # pre processing used_chans = [] template_len = template_array[seed_ids[0]].shape[1] for seed_id in seed_ids: used_chans.append(~np.isnan(template_array[seed_id]).any(axis=1)) template_array[seed_id] = ( (template_array[seed_id] - template_array[seed_id].mean(axis=-1, keepdims=True)) / ( template_array[seed_id].std(axis=-1, keepdims=True) * template_len)) template_array[seed_id] = np.nan_to_num(template_array[seed_id]) n_channels = len(seed_ids) n_templates = template_array[seed_ids[0]].shape[0] image_len = stream_array[seed_ids[0]].shape[0] fft_len = next_fast_len(template_len + image_len - 1) template_array = np.ascontiguousarray([template_array[x] for x in seed_ids], dtype=np.float32) stream_array = np.ascontiguousarray([stream_array[x] for x in seed_ids], dtype=np.float32) cccs = np.zeros((n_templates, image_len - template_len + 1), np.float32) used_chans_np = np.ascontiguousarray(used_chans, dtype=np.intc) pad_array_np = np.ascontiguousarray([pad_array[seed_id] for seed_id in seed_ids], dtype=np.intc) # call C function ret = utilslib.multi_normxcorr_fftw( template_array, n_templates, template_len, n_channels, stream_array, image_len, cccs, fft_len, used_chans_np, pad_array_np) if ret < 0: raise MemoryError() elif ret > 0: print('Error in C code (possible normalisation error)') print(cccs.max()) print(cccs.min()) raise MemoryError() return cccs, used_chans