def test_convertDatetime(self): """ Tests all time conversion methods. """ # These values are created using the Linux "date -u -d @TIMESTRING" # command. These values are assumed to be correct. timesdict = { 1234567890: UTCDateTime(2009, 2, 13, 23, 31, 30), 1111111111: UTCDateTime(2005, 3, 18, 1, 58, 31), 1212121212: UTCDateTime(2008, 5, 30, 4, 20, 12), 1313131313: UTCDateTime(2011, 8, 12, 6, 41, 53), 100000: UTCDateTime(1970, 1, 2, 3, 46, 40), 100000.111112: UTCDateTime(1970, 1, 2, 3, 46, 40, 111112), 200000000: UTCDateTime(1976, 5, 3, 19, 33, 20) } # Loop over timesdict. for ts, dt in timesdict.iteritems(): self.assertEqual(dt, util._convertMSTimeToDatetime(ts * 1000000L)) self.assertEqual(ts * 1000000L, util._convertDatetimeToMSTime(dt)) # Additional sanity tests. # Today. now = UTCDateTime() self.assertEqual(now, util._convertMSTimeToDatetime( util._convertDatetimeToMSTime(now))) # Some random date. random.seed(815) # make test reproducable timestring = random.randint(0, 2000000) * 1e6 self.assertEqual(timestring, util._convertDatetimeToMSTime( util._convertMSTimeToDatetime(timestring)))
def test_convertDatetime(self): """ Tests all time conversion methods. """ # These values are created using the Linux "date -u -d @TIMESTRING" # command. These values are assumed to be correct. timesdict = { 1234567890: UTCDateTime(2009, 2, 13, 23, 31, 30), 1111111111: UTCDateTime(2005, 3, 18, 1, 58, 31), 1212121212: UTCDateTime(2008, 5, 30, 4, 20, 12), 1313131313: UTCDateTime(2011, 8, 12, 6, 41, 53), 100000: UTCDateTime(1970, 1, 2, 3, 46, 40), 100000.111112: UTCDateTime(1970, 1, 2, 3, 46, 40, 111112), 200000000: UTCDateTime(1976, 5, 3, 19, 33, 20) } # Loop over timesdict. for ts, dt in timesdict.iteritems(): self.assertEqual(dt, util._convertMSTimeToDatetime(ts * 1000000L)) self.assertEqual(ts * 1000000L, util._convertDatetimeToMSTime(dt)) # Additional sanity tests. # Today. now = UTCDateTime() self.assertEqual( now, util._convertMSTimeToDatetime(util._convertDatetimeToMSTime(now))) # Some random date. random.seed(815) # make test reproducable timestring = random.randint(0, 2000000) * 1e6 self.assertEqual( timestring, util._convertDatetimeToMSTime( util._convertMSTimeToDatetime(timestring)))
def readMSEED(mseed_object, starttime=None, endtime=None, headonly=False, sourcename=None, reclen=None, details=False, header_byteorder=None, verbose=None, **kwargs): """ Reads a Mini-SEED file and returns a Stream object. .. warning:: This function should NOT be called directly, it registers via the ObsPy :func:`~obspy.core.stream.read` function, call this instead. :param mseed_object: Filename or open file like object that contains the binary Mini-SEED data. Any object that provides a read() method will be considered to be a file like object. :type starttime: :class:`~obspy.core.utcdatetime.UTCDateTime` :param starttime: Only read data samples after or at the start time. :type endtime: :class:`~obspy.core.utcdatetime.UTCDateTime` :param endtime: Only read data samples before or at the end time. :param headonly: Determines whether or not to unpack the data or just read the headers. :type sourcename: str :param sourcename: Source name has to have the structure 'network.station.location.channel' and can contain globbing characters. Defaults to ``None``. :param reclen: If it is None, it will be automatically determined for every record. If it is known, just set it to the record length in bytes which will increase the reading speed slightly. :type details: bool, optional :param details: If ``True`` read additional information: timing quality and availability of calibration information. Note, that the traces are then also split on these additional information. Thus the number of traces in a stream will change. Details are stored in the mseed stats AttribDict of each trace. -1 specifies for both cases, that these information is not available. ``timing_quality`` specifies the timing quality from 0 to 100 [%]. ``calibration_type`` specifies the type of available calibration information: 1 == Step Calibration, 2 == Sine Calibration, 3 == Pseudo-random Calibration, 4 == Generic Calibration and -2 == Calibration Abort. :type header_byteorder: int or str, optional :param header_byteorder: Must be either ``0`` or ``'<'`` for LSBF or little-endian, ``1`` or ``'>'`` for MBF or big-endian. ``'='`` is the native byte order. Used to enforce the header byte order. Useful in some rare cases where the automatic byte order detection fails. .. rubric:: Example >>> from obspy import read >>> st = read("/path/to/two_channels.mseed") >>> print(st) # doctest: +ELLIPSIS 2 Trace(s) in Stream: BW.UH3..EHE | 2010-06-20T00:00:00.279999Z - ... | 200.0 Hz, 386 samples BW.UH3..EHZ | 2010-06-20T00:00:00.279999Z - ... | 200.0 Hz, 386 samples >>> from obspy import UTCDateTime >>> st = read("/path/to/test.mseed", ... starttime=UTCDateTime("2003-05-29T02:16:00"), ... selection="NL.*.*.?HZ") >>> print(st) # doctest: +ELLIPSIS 1 Trace(s) in Stream: NL.HGN.00.BHZ | 2003-05-29T02:15:59.993400Z - ... | 40.0 Hz, 5629 samples """ # Parse the headonly and reclen flags. if headonly is True: unpack_data = 0 else: unpack_data = 1 if reclen is None: reclen = -1 elif reclen not in VALID_RECORD_LENGTHS: msg = 'Invalid record length. Autodetection will be used.' warnings.warn(msg) reclen = -1 # Determine the byte order. if header_byteorder == "=": header_byteorder = NATIVE_BYTEORDER if header_byteorder is None: header_byteorder = -1 elif header_byteorder in [0, "0", "<"]: header_byteorder = 0 elif header_byteorder in [1, "1", ">"]: header_byteorder = 1 # The quality flag is no more supported. Raise a warning. if 'quality' in kwargs: msg = 'The quality flag is no longer supported in this version of ' + \ 'obspy.mseed. obspy.mseed.util has some functions with similar' + \ ' behavior.' warnings.warn(msg, category=DeprecationWarning) # Parse some information about the file. if header_byteorder == 0: bo = "<" elif header_byteorder > 0: bo = ">" else: bo = None info = util.getRecordInformation(mseed_object, endian=bo) # Map the encoding to a readable string value. if info["encoding"] in ENCODINGS: info['encoding'] = ENCODINGS[info['encoding']][0] elif info["encoding"] in UNSUPPORTED_ENCODINGS: msg = ("Encoding '%s' (%i) is not supported by ObsPy. Please send " "the file to the ObsPy developers so that we can add " "support for it.") % \ (UNSUPPORTED_ENCODINGS[info['encoding']], info['encoding']) raise ValueError(msg) else: msg = "Encoding '%i' is not a valid MiniSEED encoding." % \ info['encoding'] raise ValueError(msg) # Only keep information relevant for the whole file. info = { 'encoding': info['encoding'], 'filesize': info['filesize'], 'record_length': info['record_length'], 'byteorder': info['byteorder'], 'number_of_records': info['number_of_records'] } # If its a filename just read it. if isinstance(mseed_object, (str, native_str)): # Read to NumPy array which is used as a buffer. buffer = np.fromfile(mseed_object, dtype=np.int8) elif hasattr(mseed_object, 'read'): buffer = np.fromstring(mseed_object.read(), dtype=np.int8) # Get the record length try: record_length = pow(2, int(''.join([chr(_i) for _i in buffer[19:21]]))) except ValueError: record_length = 4096 # Search for data records and pass only the data part to the underlying C # routine. offset = 0 # 0 to 9 are defined in a row in the ASCII charset. min_ascii = ord('0') # Small function to check whether an array of ASCII values contains only # digits. isdigit = lambda x: True if (x - min_ascii).max() <= 9 else False while True: # This should never happen if (isdigit(buffer[offset:offset + 6]) is False) or \ (buffer[offset + 6] not in VALID_CONTROL_HEADERS): msg = 'Not a valid (Mini-)SEED file' raise Exception(msg) elif buffer[offset + 6] in SEED_CONTROL_HEADERS: offset += record_length continue break buffer = buffer[offset:] buflen = len(buffer) # If no selection is given pass None to the C function. if starttime is None and endtime is None and sourcename is None: selections = None else: select_time = SelectTime() selections = Selections() selections.timewindows.contents = select_time if starttime is not None: if not isinstance(starttime, UTCDateTime): msg = 'starttime needs to be a UTCDateTime object' raise ValueError(msg) selections.timewindows.contents.starttime = \ util._convertDatetimeToMSTime(starttime) else: # HPTERROR results in no starttime. selections.timewindows.contents.starttime = HPTERROR if endtime is not None: if not isinstance(endtime, UTCDateTime): msg = 'endtime needs to be a UTCDateTime object' raise ValueError(msg) selections.timewindows.contents.endtime = \ util._convertDatetimeToMSTime(endtime) else: # HPTERROR results in no starttime. selections.timewindows.contents.endtime = HPTERROR if sourcename is not None: if not isinstance(sourcename, (str, native_str)): msg = 'sourcename needs to be a string' raise ValueError(msg) # libmseed uses underscores as separators and allows filtering # after the dataquality which is disabled here to not confuse # users. (* == all data qualities) selections.srcname = (sourcename.replace('.', '_') + '_*').\ encode('ascii', 'ignore') else: selections.srcname = b'*' all_data = [] # Use a callback function to allocate the memory and keep track of the # data. def allocate_data(samplecount, sampletype): # Enhanced sanity checking for libmseed 2.10 can result in the # sampletype not being set. Just return an empty array in this case. if sampletype == b"\x00": data = np.empty(0) else: data = np.empty(samplecount, dtype=DATATYPES[sampletype]) all_data.append(data) return data.ctypes.data # XXX: Do this properly! # Define Python callback function for use in C function. Return a long so # it hopefully works on 32 and 64 bit systems. allocData = C.CFUNCTYPE(C.c_long, C.c_int, C.c_char)(allocate_data) def log_error_or_warning(msg): if msg.startswith(b"ERROR: "): raise InternalMSEEDReadingError(msg[7:].strip()) if msg.startswith(b"INFO: "): msg = msg[6:].strip() # Append the offset of the full SEED header if necessary. That way # the C code does not have to deal with it. if offset and "offset" in msg: msg = ("%s The file contains a %i byte dataless part at the " "beginning. Make sure to add that to the reported " "offset to get the actual location in the file." % (msg, offset)) warnings.warn(msg, InternalMSEEDReadingWarning) diag_print = C.CFUNCTYPE(C.c_void_p, C.c_char_p)(log_error_or_warning) def log_message(msg): print(msg[6:].strip()) log_print = C.CFUNCTYPE(C.c_void_p, C.c_char_p)(log_message) try: verbose = int(verbose) except: verbose = 0 lil = clibmseed.readMSEEDBuffer(buffer, buflen, selections, C.c_int8(unpack_data), reclen, C.c_int8(verbose), C.c_int8(details), header_byteorder, allocData, diag_print, log_print) # XXX: Check if the freeing works. del selections traces = [] try: currentID = lil.contents # Return stream if not traces are found. except ValueError: clibmseed.lil_free(lil) del lil return Stream() while True: # Init header with the essential information. header = { 'network': currentID.network.strip(), 'station': currentID.station.strip(), 'location': currentID.location.strip(), 'channel': currentID.channel.strip(), 'mseed': { 'dataquality': currentID.dataquality } } # Loop over segments. try: currentSegment = currentID.firstSegment.contents except ValueError: break while True: header['sampling_rate'] = currentSegment.samprate header['starttime'] = \ util._convertMSTimeToDatetime(currentSegment.starttime) # TODO: write support is missing if details: timing_quality = currentSegment.timing_quality if timing_quality == 0xFF: # 0xFF is mask for not known timing timing_quality = -1 header['mseed']['timing_quality'] = timing_quality header['mseed']['calibration_type'] = \ currentSegment.calibration_type if headonly is False: # The data always will be in sequential order. data = all_data.pop(0) header['npts'] = len(data) else: data = np.array([]) header['npts'] = currentSegment.samplecnt # Make sure to init the number of samples. # Py3k: convert to unicode header['mseed'] = dict( (k, v.decode()) if isinstance(v, bytes) else (k, v) for k, v in header['mseed'].items()) header = dict((k, v.decode()) if isinstance(v, bytes) else (k, v) for k, v in header.items()) trace = Trace(header=header, data=data) # Append information. for key, value in info.items(): setattr(trace.stats.mseed, key, value) traces.append(trace) # A Null pointer access results in a ValueError try: currentSegment = currentSegment.next.contents except ValueError: break try: currentID = currentID.next.contents except ValueError: break clibmseed.lil_free(lil) # NOQA del lil # NOQA return Stream(traces=traces)
def readMSEED(mseed_object, starttime=None, endtime=None, headonly=False, sourcename=None, reclen=None, details=False, header_byteorder=None, verbose=None, **kwargs): """ Reads a Mini-SEED file and returns a Stream object. .. warning:: This function should NOT be called directly, it registers via the ObsPy :func:`~obspy.core.stream.read` function, call this instead. :param mseed_object: Filename or open file like object that contains the binary Mini-SEED data. Any object that provides a read() method will be considered to be a file like object. :type starttime: :class:`~obspy.core.utcdatetime.UTCDateTime` :param starttime: Only read data samples after or at the start time. :type endtime: :class:`~obspy.core.utcdatetime.UTCDateTime` :param endtime: Only read data samples before or at the end time. :param headonly: Determines whether or not to unpack the data or just read the headers. :type sourcename: str :param sourcename: Source name has to have the structure 'network.station.location.channel' and can contain globbing characters. Defaults to ``None``. :param reclen: If it is None, it will be automatically determined for every record. If it is known, just set it to the record length in bytes which will increase the reading speed slightly. :type details: bool, optional :param details: If ``True`` read additional information: timing quality and availability of calibration information. Note, that the traces are then also split on these additional information. Thus the number of traces in a stream will change. Details are stored in the mseed stats AttribDict of each trace. ``False`` specifies for both cases, that this information is not available. ``blkt1001.timing_quality`` specifies the timing quality from 0 to 100 [%]. ``calibration_type`` specifies the type of available calibration information blockettes: - ``1``: Step Calibration (Blockette 300) - ``2``: Sine Calibration (Blockette 310) - ``3``: Pseudo-random Calibration (Blockette 320) - ``4``: Generic Calibration (Blockette 390) - ``-2``: Calibration Abort (Blockette 395) :type header_byteorder: int or str, optional :param header_byteorder: Must be either ``0`` or ``'<'`` for LSBF or little-endian, ``1`` or ``'>'`` for MBF or big-endian. ``'='`` is the native byte order. Used to enforce the header byte order. Useful in some rare cases where the automatic byte order detection fails. .. rubric:: Example >>> from obspy import read >>> st = read("/path/to/two_channels.mseed") >>> print(st) # doctest: +ELLIPSIS 2 Trace(s) in Stream: BW.UH3..EHE | 2010-06-20T00:00:00.279999Z - ... | 200.0 Hz, 386 samples BW.UH3..EHZ | 2010-06-20T00:00:00.279999Z - ... | 200.0 Hz, 386 samples >>> from obspy import UTCDateTime >>> st = read("/path/to/test.mseed", ... starttime=UTCDateTime("2003-05-29T02:16:00"), ... selection="NL.*.*.?HZ") >>> print(st) # doctest: +ELLIPSIS 1 Trace(s) in Stream: NL.HGN.00.BHZ | 2003-05-29T02:15:59.993400Z - ... | 40.0 Hz, 5629 samples Read with ``details=True`` to read more details of the file if present. >>> st = read("/path/to/timingquality.mseed", details=True) >>> print(st[0].stats.mseed.blkt1001.timing_quality) 55 ``False`` means that the necessary information could not be found in the file. >>> print(st[0].stats.mseed.calibration_type) False Note that each change in timing quality from record to record may trigger a new Trace object to be created so the Stream object may contain many Trace objects if ``details=True`` is used. >>> print(len(st)) 101 """ # Parse the headonly and reclen flags. if headonly is True: unpack_data = 0 else: unpack_data = 1 if reclen is None: reclen = -1 elif reclen not in VALID_RECORD_LENGTHS: msg = 'Invalid record length. Autodetection will be used.' warnings.warn(msg) reclen = -1 # Determine the byte order. if header_byteorder == "=": header_byteorder = NATIVE_BYTEORDER if header_byteorder is None: header_byteorder = -1 elif header_byteorder in [0, "0", "<"]: header_byteorder = 0 elif header_byteorder in [1, "1", ">"]: header_byteorder = 1 # The quality flag is no more supported. Raise a warning. if 'quality' in kwargs: msg = 'The quality flag is no longer supported in this version of ' + \ 'obspy.mseed. obspy.mseed.util has some functions with similar' + \ ' behavior.' warnings.warn(msg, category=DeprecationWarning) # Parse some information about the file. if header_byteorder == 0: bo = "<" elif header_byteorder > 0: bo = ">" else: bo = None info = util.getRecordInformation(mseed_object, endian=bo) # Map the encoding to a readable string value. if info["encoding"] in ENCODINGS: info['encoding'] = ENCODINGS[info['encoding']][0] elif info["encoding"] in UNSUPPORTED_ENCODINGS: msg = ("Encoding '%s' (%i) is not supported by ObsPy. Please send " "the file to the ObsPy developers so that we can add " "support for it.") % \ (UNSUPPORTED_ENCODINGS[info['encoding']], info['encoding']) raise ValueError(msg) else: msg = "Encoding '%i' is not a valid MiniSEED encoding." % \ info['encoding'] raise ValueError(msg) # Only keep information relevant for the whole file. info = {'encoding': info['encoding'], 'filesize': info['filesize'], 'record_length': info['record_length'], 'byteorder': info['byteorder'], 'number_of_records': info['number_of_records']} # If it's a file name just read it. if isinstance(mseed_object, (str, native_str)): # Read to NumPy array which is used as a buffer. bfrNp = np.fromfile(mseed_object, dtype=np.int8) elif hasattr(mseed_object, 'read'): bfrNp = np.fromstring(mseed_object.read(), dtype=np.int8) # Get the record length try: record_length = pow(2, int(''.join([chr(_i) for _i in bfrNp[19:21]]))) except ValueError: record_length = 4096 # Search for data records and pass only the data part to the underlying C # routine. offset = 0 # 0 to 9 are defined in a row in the ASCII charset. min_ascii = ord('0') # Small function to check whether an array of ASCII values contains only # digits. isdigit = lambda x: True if (x - min_ascii).max() <= 9 else False while True: # This should never happen if (isdigit(bfrNp[offset:offset + 6]) is False) or \ (bfrNp[offset + 6] not in VALID_CONTROL_HEADERS): msg = 'Not a valid (Mini-)SEED file' raise Exception(msg) elif bfrNp[offset + 6] in SEED_CONTROL_HEADERS: offset += record_length continue break bfrNp = bfrNp[offset:] buflen = len(bfrNp) # If no selection is given pass None to the C function. if starttime is None and endtime is None and sourcename is None: selections = None else: select_time = SelectTime() selections = Selections() selections.timewindows.contents = select_time if starttime is not None: if not isinstance(starttime, UTCDateTime): msg = 'starttime needs to be a UTCDateTime object' raise ValueError(msg) selections.timewindows.contents.starttime = \ util._convertDatetimeToMSTime(starttime) else: # HPTERROR results in no starttime. selections.timewindows.contents.starttime = HPTERROR if endtime is not None: if not isinstance(endtime, UTCDateTime): msg = 'endtime needs to be a UTCDateTime object' raise ValueError(msg) selections.timewindows.contents.endtime = \ util._convertDatetimeToMSTime(endtime) else: # HPTERROR results in no starttime. selections.timewindows.contents.endtime = HPTERROR if sourcename is not None: if not isinstance(sourcename, (str, native_str)): msg = 'sourcename needs to be a string' raise ValueError(msg) # libmseed uses underscores as separators and allows filtering # after the dataquality which is disabled here to not confuse # users. (* == all data qualities) selections.srcname = (sourcename.replace('.', '_') + '_*').\ encode('ascii', 'ignore') else: selections.srcname = b'*' all_data = [] # Use a callback function to allocate the memory and keep track of the # data. def allocate_data(samplecount, sampletype): # Enhanced sanity checking for libmseed 2.10 can result in the # sampletype not being set. Just return an empty array in this case. if sampletype == b"\x00": data = np.empty(0) else: data = np.empty(samplecount, dtype=DATATYPES[sampletype]) all_data.append(data) return data.ctypes.data # XXX: Do this properly! # Define Python callback function for use in C function. Return a long so # it hopefully works on 32 and 64 bit systems. allocData = C.CFUNCTYPE(C.c_long, C.c_int, C.c_char)(allocate_data) def log_error_or_warning(msg): if msg.startswith(b"ERROR: "): raise InternalMSEEDReadingError(msg[7:].strip()) if msg.startswith(b"INFO: "): msg = msg[6:].strip() # Append the offset of the full SEED header if necessary. That way # the C code does not have to deal with it. if offset and "offset" in msg: msg = ("%s The file contains a %i byte dataless part at the " "beginning. Make sure to add that to the reported " "offset to get the actual location in the file." % ( msg, offset)) warnings.warn(msg, InternalMSEEDReadingWarning) diag_print = C.CFUNCTYPE(C.c_void_p, C.c_char_p)(log_error_or_warning) def log_message(msg): print(msg[6:].strip()) log_print = C.CFUNCTYPE(C.c_void_p, C.c_char_p)(log_message) try: verbose = int(verbose) except: verbose = 0 lil = clibmseed.readMSEEDBuffer( bfrNp, buflen, selections, C.c_int8(unpack_data), reclen, C.c_int8(verbose), C.c_int8(details), header_byteorder, allocData, diag_print, log_print) # XXX: Check if the freeing works. del selections traces = [] try: currentID = lil.contents # Return stream if not traces are found. except ValueError: clibmseed.lil_free(lil) del lil return Stream() while True: # Init header with the essential information. header = {'network': currentID.network.strip(), 'station': currentID.station.strip(), 'location': currentID.location.strip(), 'channel': currentID.channel.strip(), 'mseed': {'dataquality': currentID.dataquality}} # Loop over segments. try: currentSegment = currentID.firstSegment.contents except ValueError: break while True: header['sampling_rate'] = currentSegment.samprate header['starttime'] = \ util._convertMSTimeToDatetime(currentSegment.starttime) if details: timing_quality = currentSegment.timing_quality if timing_quality == 0xFF: # 0xFF is mask for not known timing timing_quality = False header['mseed']['blkt1001'] = {} header['mseed']['blkt1001']['timing_quality'] = timing_quality header['mseed']['calibration_type'] = \ currentSegment.calibration_type \ if currentSegment.calibration_type != -1 else False if headonly is False: # The data always will be in sequential order. data = all_data.pop(0) header['npts'] = len(data) else: data = np.array([]) header['npts'] = currentSegment.samplecnt # Make sure to init the number of samples. # Py3k: convert to unicode header['mseed'] = dict((k, v.decode()) if isinstance(v, bytes) else (k, v) for k, v in header['mseed'].items()) header = dict((k, v.decode()) if isinstance(v, bytes) else (k, v) for k, v in header.items()) trace = Trace(header=header, data=data) # Append information. for key, value in info.items(): setattr(trace.stats.mseed, key, value) traces.append(trace) # A Null pointer access results in a ValueError try: currentSegment = currentSegment.next.contents except ValueError: break try: currentID = currentID.next.contents except ValueError: break clibmseed.lil_free(lil) # NOQA del lil # NOQA return Stream(traces=traces)