Example #1
0
File: core.py Project: egdorf/obspy
def readMSEED(
    mseed_object, starttime=None, endtime=None, headonly=False, sourcename=None, reclen=None, recinfo=True, **kwargs
):
    """
    Reads a Mini-SEED file and returns a Stream object.

    .. warning::
        This function should NOT be called directly, it registers via the
        ObsPy :func:`~obspy.core.stream.read` function, call this instead.

    :param mseed_object: Filename or open file like object that contains the
        binary Mini-SEED data. Any object that provides a read() method will be
        considered to be a file like object.
    :type starttime: UTCDateTime
    :param starttime: Only read data samples after or at the starttime.
    :type endtime: UTCDateTime
    :param endtime: Only read data samples before or at the starttime.
    :param headonly: Determines whether or not to unpack the data or just
        read the headers.
    :type sourcename: str
    :param sourcename: Sourcename has to have the structure
        'network.station.location.channel' and can contain globbing characters.
        Defaults to ``None``.
    :param reclen: If it is None, it will be automatically determined for every
        record. If it is known, just set it to the record length in bytes which
        will increase the reading speed slightly.
    :type recinfo: bool, optional
    :param recinfo: If ``True`` the byteorder, record length and the
        encoding of the file will be read and stored in every Trace's
        stats.mseed AttribDict. These stored attributes will also be used while
        writing a Mini-SEED file. Only the very first record of the file will
        be read and all following records are assumed to be the same. Defaults
        to ``True``.

    .. rubric:: Example

    >>> from obspy.core import read
    >>> st = read("/path/to/two_channels.mseed")
    >>> print(st)  # doctest: +ELLIPSIS
    2 Trace(s) in Stream:
    BW.UH3..EHE | 2010-06-20T00:00:00.279999Z - ... | 200.0 Hz, 386 samples
    BW.UH3..EHZ | 2010-06-20T00:00:00.279999Z - ... | 200.0 Hz, 386 samples

    >>> from obspy.core import UTCDateTime
    >>> st = read("/path/to/test.mseed",
    ...           starttime=UTCDateTime("2003-05-29T02:16:00"),
    ...           selection="NL.*.*.?HZ")
    >>> print(st)  # doctest: +ELLIPSIS
    1 Trace(s) in Stream:
    NL.HGN.00.BHZ | 2003-05-29T02:15:59.993400Z - ... | 40.0 Hz, 5629 samples
    """
    # Parse the headonly and reclen flags.
    if headonly is True:
        unpack_data = 0
    else:
        unpack_data = 1
    if reclen is None:
        reclen = -1
    elif reclen is not None and reclen not in VALID_RECORD_LENGTHS:
        msg = "Invalid record length. Autodetection will be used."
        warnings.warn(msg)
        reclen = -1
    else:
        reclen = int(log(reclen, 2))

    # The quality flag is no more supported. Raise a warning.
    if "quality" in kwargs:
        msg = (
            "The quality flag is no more supported in this version of "
            + "obspy.mseed. obspy.mseed.util has some functions with similar "
            + "behaviour."
        )
        warnings.warn(msg, category=DeprecationWarning)

    # Parse some information about the file.
    if recinfo:
        info = util.getRecordInformation(mseed_object)
        info["encoding"] = ENCODINGS[info["encoding"]][0]
        # Only keep information relevant for the whole file.
        info = {
            "encoding": info["encoding"],
            "filesize": info["filesize"],
            "record_length": info["record_length"],
            "byteorder": info["byteorder"],
            "number_of_records": info["number_of_records"],
        }

    # If its a filename just read it.
    if isinstance(mseed_object, basestring):
        # Read to NumPy array which is used as a buffer.
        buffer = np.fromfile(mseed_object, dtype="b")
    elif hasattr(mseed_object, "read"):
        buffer = np.fromstring(mseed_object.read(), dtype="b")

    # Get the record length
    try:
        record_length = pow(2, int("".join([chr(_i) for _i in buffer[19:21]])))
    except ValueError:
        record_length = 4096

    # Search for data records and pass only the data part to the underlying C
    # routine.
    offset = 0
    # 0 to 9 are defined in a row in the ASCII charset.
    min_ascii = ord("0")
    # Small function to check whether an array of ASCII values contains only
    # digits.
    isdigit = lambda x: True if (x - min_ascii).max() <= 9 else False
    while True:
        # This should never happen
        if (isdigit(buffer[offset : offset + 6]) is False) or (buffer[offset + 6] not in VALID_CONTROL_HEADERS):
            msg = "Not a valid (Mini-)SEED file"
            raise Exception(msg)
        elif buffer[offset + 6] in SEED_CONTROL_HEADERS:
            offset += record_length
            continue
        break
    buffer = buffer[offset:]
    buflen = len(buffer)

    # If no selection is given pass None to the C function.
    if starttime is None and endtime is None and sourcename is None:
        selections = None
    else:
        select_time = SelectTime()
        selections = Selections()
        selections.timewindows.contents = select_time
        if starttime is not None:
            if not isinstance(starttime, UTCDateTime):
                msg = "starttime needs to be a UTCDateTime object"
                raise ValueError(msg)
            selections.timewindows.contents.starttime = util._convertDatetimeToMSTime(starttime)
        else:
            # HPTERROR results in no starttime.
            selections.timewindows.contents.starttime = HPTERROR
        if endtime is not None:
            if not isinstance(endtime, UTCDateTime):
                msg = "endtime needs to be a UTCDateTime object"
                raise ValueError(msg)
            selections.timewindows.contents.endtime = util._convertDatetimeToMSTime(endtime)
        else:
            # HPTERROR results in no starttime.
            selections.timewindows.contents.endtime = HPTERROR
        if sourcename is not None:
            if not isinstance(sourcename, basestring):
                msg = "sourcename needs to be a string"
                raise ValueError(msg)
            # libmseed uses underscores as separators and allows filtering
            # after the dataquality which is disabled here to not confuse
            # users. (* == all data qualities)
            selections.srcname = sourcename.replace(".", "_") + "_*"
        else:
            selections.srcname = "*"
    all_data = []

    # Use a callback function to allocate the memory and keep track of the
    # data.
    def allocate_data(samplecount, sampletype):
        data = np.empty(samplecount, dtype=DATATYPES[sampletype])
        all_data.append(data)
        return data.ctypes.data

    # XXX: Do this properly!
    # Define Python callback function for use in C function. Return a long so
    # it hopefully works on 32 and 64 bit systems.
    allocData = C.CFUNCTYPE(C.c_long, C.c_int, C.c_char)(allocate_data)

    lil = clibmseed.readMSEEDBuffer(buffer, buflen, selections, unpack_data, reclen, 0, allocData)

    # XXX: Check if the freeing works.
    del selections

    traces = []
    try:
        currentID = lil.contents
    # Return stream if not traces are found.
    except ValueError:
        clibmseed.lil_free(lil)
        del lil
        return Stream()

    while True:
        # Init header with the essential information.
        header = {
            "network": currentID.network.strip(),
            "station": currentID.station.strip(),
            "location": currentID.location.strip(),
            "channel": currentID.channel.strip(),
            "mseed": {"dataquality": currentID.dataquality},
        }
        # Loop over segments.
        try:
            currentSegment = currentID.firstSegment.contents
        except ValueError:
            break
        while True:
            header["sampling_rate"] = currentSegment.samprate
            header["starttime"] = util._convertMSTimeToDatetime(currentSegment.starttime)
            if headonly is False:
                # The data always will be in sequential order.
                data = all_data.pop(0)
                header["npts"] = len(data)
            else:
                data = np.array([])
                header["npts"] = currentSegment.samplecnt
            # Make sure to init the number of samples.
            trace = Trace(header=header, data=data)
            # Append information if necessary.
            if recinfo:
                for key, value in info.iteritems():
                    setattr(trace.stats.mseed, key, value)
            traces.append(trace)
            # A Null pointer access results in a ValueError
            try:
                currentSegment = currentSegment.next.contents
            except ValueError:
                break
        try:
            currentID = currentID.next.contents
        except ValueError:
            break

    clibmseed.lil_free(lil)
    del lil
    return Stream(traces=traces)
Example #2
0
def readMSEED(mseed_object,
              starttime=None,
              endtime=None,
              headonly=False,
              sourcename=None,
              reclen=None,
              recinfo=True,
              details=False,
              header_byteorder=None,
              verbose=None,
              **kwargs):
    """
    Reads a Mini-SEED file and returns a Stream object.

    .. warning::
        This function should NOT be called directly, it registers via the
        ObsPy :func:`~obspy.core.stream.read` function, call this instead.

    :param mseed_object: Filename or open file like object that contains the
        binary Mini-SEED data. Any object that provides a read() method will be
        considered to be a file like object.
    :type starttime: UTCDateTime
    :param starttime: Only read data samples after or at the starttime.
    :type endtime: UTCDateTime
    :param endtime: Only read data samples before or at the starttime.
    :param headonly: Determines whether or not to unpack the data or just
        read the headers.
    :type sourcename: str
    :param sourcename: Sourcename has to have the structure
        'network.station.location.channel' and can contain globbing characters.
        Defaults to ``None``.
    :param reclen: If it is None, it will be automatically determined for every
        record. If it is known, just set it to the record length in bytes which
        will increase the reading speed slightly.
    :type recinfo: bool, optional
    :param recinfo: If ``True`` the byteorder, record length and the
        encoding of the file will be read and stored in every Trace's
        stats.mseed AttribDict. These stored attributes will also be used while
        writing a Mini-SEED file. Only the very first record of the file will
        be read and all following records are assumed to be the same. Defaults
        to ``True``.
    :type details: bool, optional
    :param details: If ``True`` read additional information: timing quality
        and availability of calibration information.
        Note, that the traces are then also split on these additional
        information. Thus the number of traces in a stream will change.
        Details are stored in the mseed stats AttribDict of each trace.
        -1 specifies for both cases, that these information is not available.
        ``timing_quality`` specifies the timing quality from 0 to 100 [%].
        ``calibration_type`` specifies the type of available calibration
        information: 1 == Step Calibration, 2 == Sine Calibration, 3 ==
        Pseudo-random Calibration, 4 == Generic Calibration and -2 ==
        Calibration Abort.
    :type header_byteorder: [``0`` or ``'<'`` | ``1`` or ``'>'`` | ``'='``],
        optional
    :param header_byteorder: Must be either ``0`` or ``'<'`` for LSBF or
        little-endian, ``1`` or ``'>'`` for MBF or big-endian. ``'='`` is the
        native byteorder. Used to enforce the header byteorder. Useful in some
        rare cases where the automatic byte order detection fails.

    .. rubric:: Example

    >>> from obspy import read
    >>> st = read("/path/to/two_channels.mseed")
    >>> print(st)  # doctest: +ELLIPSIS
    2 Trace(s) in Stream:
    BW.UH3..EHE | 2010-06-20T00:00:00.279999Z - ... | 200.0 Hz, 386 samples
    BW.UH3..EHZ | 2010-06-20T00:00:00.279999Z - ... | 200.0 Hz, 386 samples

    >>> from obspy import UTCDateTime
    >>> st = read("/path/to/test.mseed",
    ...           starttime=UTCDateTime("2003-05-29T02:16:00"),
    ...           selection="NL.*.*.?HZ")
    >>> print(st)  # doctest: +ELLIPSIS
    1 Trace(s) in Stream:
    NL.HGN.00.BHZ | 2003-05-29T02:15:59.993400Z - ... | 40.0 Hz, 5629 samples
    """
    # Parse the headonly and reclen flags.
    if headonly is True:
        unpack_data = 0
    else:
        unpack_data = 1
    if reclen is None:
        reclen = -1
    elif reclen not in VALID_RECORD_LENGTHS:
        msg = 'Invalid record length. Autodetection will be used.'
        warnings.warn(msg)
        reclen = -1

    # Determine the byteorder.
    if header_byteorder == "=":
        header_byteorder = NATIVE_BYTEORDER

    if header_byteorder is None:
        header_byteorder = -1
    elif header_byteorder in [0, "0", "<"]:
        header_byteorder = 0
    elif header_byteorder in [1, "1", ">"]:
        header_byteorder = 1

    # The quality flag is no more supported. Raise a warning.
    if 'quality' in kwargs:
        msg = 'The quality flag is no more supported in this version of ' + \
            'obspy.mseed. obspy.mseed.util has some functions with similar' + \
            ' behaviour.'
        warnings.warn(msg, category=DeprecationWarning)

    # Parse some information about the file.
    if recinfo:
        # Pass the byteorder if enforced.
        if header_byteorder == 0:
            bo = "<"
        elif header_byteorder > 0:
            bo = ">"
        else:
            bo = None

        info = util.getRecordInformation(mseed_object, endian=bo)
        info['encoding'] = ENCODINGS[info['encoding']][0]
        # Only keep information relevant for the whole file.
        info = {
            'encoding': info['encoding'],
            'filesize': info['filesize'],
            'record_length': info['record_length'],
            'byteorder': info['byteorder'],
            'number_of_records': info['number_of_records']
        }

    # If its a filename just read it.
    if isinstance(mseed_object, basestring):
        # Read to NumPy array which is used as a buffer.
        buffer = np.fromfile(mseed_object, dtype='b')
    elif hasattr(mseed_object, 'read'):
        buffer = np.fromstring(mseed_object.read(), dtype='b')

    # Get the record length
    try:
        record_length = pow(2, int(''.join([chr(_i) for _i in buffer[19:21]])))
    except ValueError:
        record_length = 4096

    # Search for data records and pass only the data part to the underlying C
    # routine.
    offset = 0
    # 0 to 9 are defined in a row in the ASCII charset.
    min_ascii = ord('0')
    # Small function to check whether an array of ASCII values contains only
    # digits.
    isdigit = lambda x: True if (x - min_ascii).max() <= 9 else False
    while True:
        # This should never happen
        if (isdigit(buffer[offset:offset + 6]) is False) or \
                (buffer[offset + 6] not in VALID_CONTROL_HEADERS):
            msg = 'Not a valid (Mini-)SEED file'
            raise Exception(msg)
        elif buffer[offset + 6] in SEED_CONTROL_HEADERS:
            offset += record_length
            continue
        break
    buffer = buffer[offset:]
    buflen = len(buffer)

    # If no selection is given pass None to the C function.
    if starttime is None and endtime is None and sourcename is None:
        selections = None
    else:
        select_time = SelectTime()
        selections = Selections()
        selections.timewindows.contents = select_time
        if starttime is not None:
            if not isinstance(starttime, UTCDateTime):
                msg = 'starttime needs to be a UTCDateTime object'
                raise ValueError(msg)
            selections.timewindows.contents.starttime = \
                util._convertDatetimeToMSTime(starttime)
        else:
            # HPTERROR results in no starttime.
            selections.timewindows.contents.starttime = HPTERROR
        if endtime is not None:
            if not isinstance(endtime, UTCDateTime):
                msg = 'endtime needs to be a UTCDateTime object'
                raise ValueError(msg)
            selections.timewindows.contents.endtime = \
                util._convertDatetimeToMSTime(endtime)
        else:
            # HPTERROR results in no starttime.
            selections.timewindows.contents.endtime = HPTERROR
        if sourcename is not None:
            if not isinstance(sourcename, basestring):
                msg = 'sourcename needs to be a string'
                raise ValueError(msg)
            # libmseed uses underscores as separators and allows filtering
            # after the dataquality which is disabled here to not confuse
            # users. (* == all data qualities)
            selections.srcname = sourcename.replace('.', '_') + '_*'
        else:
            selections.srcname = '*'
    all_data = []

    # Use a callback function to allocate the memory and keep track of the
    # data.
    def allocate_data(samplecount, sampletype):
        # Enhanced sanity checking for libmseed 2.10 can result in the
        # sampletype not being set. Just return an empty array in this case.
        if sampletype == "\x00":
            data = np.empty(0)
        else:
            data = np.empty(samplecount, dtype=DATATYPES[sampletype])
        all_data.append(data)
        return data.ctypes.data

    # XXX: Do this properly!
    # Define Python callback function for use in C function. Return a long so
    # it hopefully works on 32 and 64 bit systems.
    allocData = C.CFUNCTYPE(C.c_long, C.c_int, C.c_char)(allocate_data)

    def log_error_or_warning(msg):
        if msg.startswith("ERROR: "):
            raise InternalMSEEDReadingError(msg[7:].strip())
        if msg.startswith("INFO: "):
            warnings.warn(msg[6:].strip(), InternalMSEEDReadingWarning)

    diag_print = C.CFUNCTYPE(C.c_void_p, C.c_char_p)(log_error_or_warning)

    def log_message(msg):
        print msg[6:].strip()

    log_print = C.CFUNCTYPE(C.c_void_p, C.c_char_p)(log_message)

    try:
        verbose = int(verbose)
    except:
        verbose = 0

    lil = clibmseed.readMSEEDBuffer(buffer, buflen, selections,
                                    C.c_int8(unpack_data), reclen,
                                    C.c_int8(verbose), C.c_int8(details),
                                    header_byteorder, allocData, diag_print,
                                    log_print)

    # XXX: Check if the freeing works.
    del selections

    traces = []
    try:
        currentID = lil.contents
    # Return stream if not traces are found.
    except ValueError:
        clibmseed.lil_free(lil)
        del lil
        return Stream()

    while True:
        # Init header with the essential information.
        header = {
            'network': currentID.network.strip(),
            'station': currentID.station.strip(),
            'location': currentID.location.strip(),
            'channel': currentID.channel.strip(),
            'mseed': {
                'dataquality': currentID.dataquality
            }
        }
        # Loop over segments.
        try:
            currentSegment = currentID.firstSegment.contents
        except ValueError:
            break
        while True:
            header['sampling_rate'] = currentSegment.samprate
            header['starttime'] = \
                util._convertMSTimeToDatetime(currentSegment.starttime)
            # TODO: write support is missing
            if details:
                timing_quality = currentSegment.timing_quality
                if timing_quality == 0xFF:  # 0xFF is mask for not known timing
                    timing_quality = -1
                header['mseed']['timing_quality'] = timing_quality
                header['mseed']['calibration_type'] = \
                    currentSegment.calibration_type

            if headonly is False:
                # The data always will be in sequential order.
                data = all_data.pop(0)
                header['npts'] = len(data)
            else:
                data = np.array([])
                header['npts'] = currentSegment.samplecnt
            # Make sure to init the number of samples.
            trace = Trace(header=header, data=data)
            # Append information if necessary.
            if recinfo:
                for key, value in info.iteritems():
                    setattr(trace.stats.mseed, key, value)
            traces.append(trace)
            # A Null pointer access results in a ValueError
            try:
                currentSegment = currentSegment.next.contents
            except ValueError:
                break
        try:
            currentID = currentID.next.contents
        except ValueError:
            break

    clibmseed.lil_free(lil)  # NOQA
    del lil  # NOQA
    return Stream(traces=traces)
Example #3
0
def readMSEED(mseed_object,
              starttime=None,
              endtime=None,
              headonly=False,
              sourcename=None,
              reclen=None,
              recinfo=True,
              **kwargs):
    """
    Reads a Mini-SEED file and returns a Stream object.

    .. warning::
        This function should NOT be called directly, it registers via the
        ObsPy :func:`~obspy.core.stream.read` function, call this instead.

    :param mseed_object: Filename or open file like object that contains the
        binary Mini-SEED data. Any object that provides a read() method will be
        considered to be a file like object.
    :type starttime: UTCDateTime
    :param starttime: Only read data samples after or at the starttime.
    :type endtime: UTCDateTime
    :param endtime: Only read data samples before or at the starttime.
    :param headonly: Determines whether or not to unpack the data or just
        read the headers.
    :type sourcename: str
    :param sourcename: Sourcename has to have the structure
        'network.station.location.channel' and can contain globbing characters.
        Defaults to ``None``.
    :param reclen: If it is None, it will be automatically determined for every
        record. If it is known, just set it to the record length in bytes which
        will increase the reading speed slightly.
    :type recinfo: bool, optional
    :param recinfo: If ``True`` the byteorder, record length and the
        encoding of the file will be read and stored in every Trace's
        stats.mseed AttribDict. These stored attributes will also be used while
        writing a Mini-SEED file. Only the very first record of the file will
        be read and all following records are assumed to be the same. Defaults
        to ``True``.

    .. rubric:: Example

    >>> from obspy.core import read
    >>> st = read("/path/to/two_channels.mseed")
    >>> print(st)  # doctest: +ELLIPSIS
    2 Trace(s) in Stream:
    BW.UH3..EHE | 2010-06-20T00:00:00.279999Z - ... | 200.0 Hz, 386 samples
    BW.UH3..EHZ | 2010-06-20T00:00:00.279999Z - ... | 200.0 Hz, 386 samples

    >>> from obspy.core import UTCDateTime
    >>> st = read("/path/to/test.mseed",
    ...           starttime=UTCDateTime("2003-05-29T02:16:00"),
    ...           selection="NL.*.*.?HZ")
    >>> print(st)  # doctest: +ELLIPSIS
    1 Trace(s) in Stream:
    NL.HGN.00.BHZ | 2003-05-29T02:15:59.993400Z - ... | 40.0 Hz, 5629 samples
    """
    # Parse the headonly and reclen flags.
    if headonly is True:
        unpack_data = 0
    else:
        unpack_data = 1
    if reclen is None:
        reclen = -1
    elif reclen is not None and reclen not in VALID_RECORD_LENGTHS:
        msg = 'Invalid record length. Autodetection will be used.'
        warnings.warn(msg)
        reclen = -1
    else:
        reclen = int(log(reclen, 2))

    # The quality flag is no more supported. Raise a warning.
    if 'quality' in kwargs:
        msg = 'The quality flag is no more supported in this version of ' + \
        'obspy.mseed. obspy.mseed.util has some functions with similar ' + \
        'behaviour.'
        warnings.warn(msg, category=DeprecationWarning)

    # Parse some information about the file.
    if recinfo:
        info = util.getRecordInformation(mseed_object)
        info['encoding'] = ENCODINGS[info['encoding']][0]
        # Only keep information relevant for the whole file.
        info = {
            'encoding': info['encoding'],
            'filesize': info['filesize'],
            'record_length': info['record_length'],
            'byteorder': info['byteorder'],
            'number_of_records': info['number_of_records']
        }

    # If its a filename just read it.
    if isinstance(mseed_object, basestring):
        # Read to NumPy array which is used as a buffer.
        buffer = np.fromfile(mseed_object, dtype='b')
    elif hasattr(mseed_object, 'read'):
        buffer = np.fromstring(mseed_object.read(), dtype='b')

    # Get the record length
    try:
        record_length = pow(2, int(''.join([chr(_i) for _i in buffer[19:21]])))
    except ValueError:
        record_length = 4096

    # Search for data records and pass only the data part to the underlying C
    # routine.
    offset = 0
    # 0 to 9 are defined in a row in the ASCII charset.
    min_ascii = ord('0')
    # Small function to check whether an array of ASCII values contains only
    # digits.
    isdigit = lambda x: True if (x - min_ascii).max() <= 9 else False
    while True:
        # This should never happen
        if (isdigit(buffer[offset:offset + 6]) is False) or \
            (buffer[offset + 6] not in VALID_CONTROL_HEADERS):
            msg = 'Not a valid (Mini-)SEED file'
            raise Exception(msg)
        elif buffer[offset + 6] in SEED_CONTROL_HEADERS:
            offset += record_length
            continue
        break
    buffer = buffer[offset:]
    buflen = len(buffer)

    # If no selection is given pass None to the C function.
    if starttime is None and endtime is None and sourcename is None:
        selections = None
    else:
        select_time = SelectTime()
        selections = Selections()
        selections.timewindows.contents = select_time
        if starttime is not None:
            if not isinstance(starttime, UTCDateTime):
                msg = 'starttime needs to be a UTCDateTime object'
                raise ValueError(msg)
            selections.timewindows.contents.starttime = \
                util._convertDatetimeToMSTime(starttime)
        else:
            # HPTERROR results in no starttime.
            selections.timewindows.contents.starttime = HPTERROR
        if endtime is not None:
            if not isinstance(endtime, UTCDateTime):
                msg = 'endtime needs to be a UTCDateTime object'
                raise ValueError(msg)
            selections.timewindows.contents.endtime = \
                util._convertDatetimeToMSTime(endtime)
        else:
            # HPTERROR results in no starttime.
            selections.timewindows.contents.endtime = HPTERROR
        if sourcename is not None:
            if not isinstance(sourcename, basestring):
                msg = 'sourcename needs to be a string'
                raise ValueError(msg)
            # libmseed uses underscores as separators and allows filtering
            # after the dataquality which is disabled here to not confuse
            # users. (* == all data qualities)
            selections.srcname = sourcename.replace('.', '_') + '_*'
        else:
            selections.srcname = '*'
    all_data = []

    # Use a callback function to allocate the memory and keep track of the
    # data.
    def allocate_data(samplecount, sampletype):
        data = np.empty(samplecount, dtype=DATATYPES[sampletype])
        all_data.append(data)
        return data.ctypes.data

    # XXX: Do this properly!
    # Define Python callback function for use in C function. Return a long so
    # it hopefully works on 32 and 64 bit systems.
    allocData = C.CFUNCTYPE(C.c_long, C.c_int, C.c_char)(allocate_data)

    lil = clibmseed.readMSEEDBuffer(buffer, buflen, selections, unpack_data,
                                    reclen, 0, allocData)

    # XXX: Check if the freeing works.
    del selections

    traces = []
    try:
        currentID = lil.contents
    # Return stream if not traces are found.
    except ValueError:
        clibmseed.lil_free(lil)
        del lil
        return Stream()

    while True:
        # Init header with the essential information.
        header = {
            'network': currentID.network.strip(),
            'station': currentID.station.strip(),
            'location': currentID.location.strip(),
            'channel': currentID.channel.strip(),
            'mseed': {
                'dataquality': currentID.dataquality
            }
        }
        # Loop over segments.
        try:
            currentSegment = currentID.firstSegment.contents
        except ValueError:
            break
        while True:
            header['sampling_rate'] = currentSegment.samprate
            header['starttime'] = \
                util._convertMSTimeToDatetime(currentSegment.starttime)
            if headonly is False:
                # The data always will be in sequential order.
                data = all_data.pop(0)
                header['npts'] = len(data)
            else:
                data = np.array([])
                header['npts'] = currentSegment.samplecnt
            # Make sure to init the number of samples.
            trace = Trace(header=header, data=data)
            # Append information if necessary.
            if recinfo:
                for key, value in info.iteritems():
                    setattr(trace.stats.mseed, key, value)
            traces.append(trace)
            # A Null pointer access results in a ValueError
            try:
                currentSegment = currentSegment.next.contents
            except ValueError:
                break
        try:
            currentID = currentID.next.contents
        except ValueError:
            break

    clibmseed.lil_free(lil)
    del lil
    return Stream(traces=traces)
Example #4
0
def readMSEED(mseed_object, starttime=None, endtime=None, headonly=False,
              sourcename=None, reclen=None, recinfo=True, details=False,
              header_byteorder=None, verbose=None, **kwargs):
    """
    Reads a Mini-SEED file and returns a Stream object.

    .. warning::
        This function should NOT be called directly, it registers via the
        ObsPy :func:`~obspy.core.stream.read` function, call this instead.

    :param mseed_object: Filename or open file like object that contains the
        binary Mini-SEED data. Any object that provides a read() method will be
        considered to be a file like object.
    :type starttime: UTCDateTime
    :param starttime: Only read data samples after or at the starttime.
    :type endtime: UTCDateTime
    :param endtime: Only read data samples before or at the starttime.
    :param headonly: Determines whether or not to unpack the data or just
        read the headers.
    :type sourcename: str
    :param sourcename: Sourcename has to have the structure
        'network.station.location.channel' and can contain globbing characters.
        Defaults to ``None``.
    :param reclen: If it is None, it will be automatically determined for every
        record. If it is known, just set it to the record length in bytes which
        will increase the reading speed slightly.
    :type recinfo: bool, optional
    :param recinfo: If ``True`` the byteorder, record length and the
        encoding of the file will be read and stored in every Trace's
        stats.mseed AttribDict. These stored attributes will also be used while
        writing a Mini-SEED file. Only the very first record of the file will
        be read and all following records are assumed to be the same. Defaults
        to ``True``.
    :type details: bool, optional
    :param details: If ``True`` read additional information: timing quality
        and availability of calibration information.
        Note, that the traces are then also split on these additional
        information. Thus the number of traces in a stream will change.
        Details are stored in the mseed stats AttribDict of each trace.
        -1 specifies for both cases, that these information is not available.
        ``timing_quality`` specifies the timing quality from 0 to 100 [%].
        ``calibration_type`` specifies the type of available calibration
        information: 1 == Step Calibration, 2 == Sine Calibration, 3 ==
        Pseudo-random Calibration, 4 == Generic Calibration and -2 ==
        Calibration Abort.
    :type header_byteorder: [``0`` or ``'<'`` | ``1`` or ``'>'`` | ``'='``],
        optional
    :param header_byteorder: Must be either ``0`` or ``'<'`` for LSBF or
        little-endian, ``1`` or ``'>'`` for MBF or big-endian. ``'='`` is the
        native byteorder. Used to enforce the header byteorder. Useful in some
        rare cases where the automatic byte order detection fails.

    .. rubric:: Example

    >>> from obspy import read
    >>> st = read("/path/to/two_channels.mseed")
    >>> print(st)  # doctest: +ELLIPSIS
    2 Trace(s) in Stream:
    BW.UH3..EHE | 2010-06-20T00:00:00.279999Z - ... | 200.0 Hz, 386 samples
    BW.UH3..EHZ | 2010-06-20T00:00:00.279999Z - ... | 200.0 Hz, 386 samples

    >>> from obspy import UTCDateTime
    >>> st = read("/path/to/test.mseed",
    ...           starttime=UTCDateTime("2003-05-29T02:16:00"),
    ...           selection="NL.*.*.?HZ")
    >>> print(st)  # doctest: +ELLIPSIS
    1 Trace(s) in Stream:
    NL.HGN.00.BHZ | 2003-05-29T02:15:59.993400Z - ... | 40.0 Hz, 5629 samples
    """
    # Parse the headonly and reclen flags.
    if headonly is True:
        unpack_data = 0
    else:
        unpack_data = 1
    if reclen is None:
        reclen = -1
    elif reclen is not None and reclen not in VALID_RECORD_LENGTHS:
        msg = 'Invalid record length. Autodetection will be used.'
        warnings.warn(msg)
        reclen = -1
    else:
        reclen = int(log(reclen, 2))

    # Determine the byteorder.
    if header_byteorder == "=":
        header_byteorder = NATIVE_BYTEORDER

    if header_byteorder is None:
        header_byteorder = -1
    elif header_byteorder in [0, "0", "<"]:
        header_byteorder = 0
    elif header_byteorder in [1, "1", ">"]:
        header_byteorder = 1

    # The quality flag is no more supported. Raise a warning.
    if 'quality' in kwargs:
        msg = 'The quality flag is no more supported in this version of ' + \
        'obspy.mseed. obspy.mseed.util has some functions with similar ' + \
        'behaviour.'
        warnings.warn(msg, category=DeprecationWarning)

    # Parse some information about the file.
    if recinfo:
        # Pass the byteorder if enforced.
        if header_byteorder == 0:
            bo = "<"
        elif header_byteorder > 0:
            bo = ">"
        else:
            bo = None

        info = util.getRecordInformation(mseed_object, endian=bo)
        info['encoding'] = ENCODINGS[info['encoding']][0]
        # Only keep information relevant for the whole file.
        info = {'encoding': info['encoding'],
                'filesize': info['filesize'],
                'record_length': info['record_length'],
                'byteorder': info['byteorder'],
                'number_of_records': info['number_of_records']}

    # If its a filename just read it.
    if isinstance(mseed_object,  basestring):
        # Read to NumPy array which is used as a buffer.
        buffer = np.fromfile(mseed_object, dtype='b')
    elif hasattr(mseed_object, 'read'):
        buffer = np.fromstring(mseed_object.read(), dtype='b')

    # Get the record length
    try:
        record_length = pow(2, int(''.join([chr(_i) for _i in buffer[19:21]])))
    except ValueError:
        record_length = 4096

    # Search for data records and pass only the data part to the underlying C
    # routine.
    offset = 0
    # 0 to 9 are defined in a row in the ASCII charset.
    min_ascii = ord('0')
    # Small function to check whether an array of ASCII values contains only
    # digits.
    isdigit = lambda x: True if (x - min_ascii).max() <= 9 else False
    while True:
        # This should never happen
        if (isdigit(buffer[offset:offset + 6]) is False) or \
            (buffer[offset + 6] not in VALID_CONTROL_HEADERS):
            msg = 'Not a valid (Mini-)SEED file'
            raise Exception(msg)
        elif buffer[offset + 6] in SEED_CONTROL_HEADERS:
            offset += record_length
            continue
        break
    buffer = buffer[offset:]
    buflen = len(buffer)

    # If no selection is given pass None to the C function.
    if starttime is None and endtime is None and sourcename is None:
        selections = None
    else:
        select_time = SelectTime()
        selections = Selections()
        selections.timewindows.contents = select_time
        if starttime is not None:
            if not isinstance(starttime, UTCDateTime):
                msg = 'starttime needs to be a UTCDateTime object'
                raise ValueError(msg)
            selections.timewindows.contents.starttime = \
                util._convertDatetimeToMSTime(starttime)
        else:
            # HPTERROR results in no starttime.
            selections.timewindows.contents.starttime = HPTERROR
        if endtime is not None:
            if not isinstance(endtime, UTCDateTime):
                msg = 'endtime needs to be a UTCDateTime object'
                raise ValueError(msg)
            selections.timewindows.contents.endtime = \
                util._convertDatetimeToMSTime(endtime)
        else:
            # HPTERROR results in no starttime.
            selections.timewindows.contents.endtime = HPTERROR
        if sourcename is not None:
            if not isinstance(sourcename, basestring):
                msg = 'sourcename needs to be a string'
                raise ValueError(msg)
            # libmseed uses underscores as separators and allows filtering
            # after the dataquality which is disabled here to not confuse
            # users. (* == all data qualities)
            selections.srcname = sourcename.replace('.', '_') + '_*'
        else:
            selections.srcname = '*'
    all_data = []

    # Use a callback function to allocate the memory and keep track of the
    # data.
    def allocate_data(samplecount, sampletype):
        # Enhanced sanity checking for libmseed 2.10 can result in the
        # sampletype not being set. Just return an empty array in this case.
        if sampletype == "\x00":
            data = np.empty(0)
        else:
            data = np.empty(samplecount, dtype=DATATYPES[sampletype])
        all_data.append(data)
        return data.ctypes.data
    # XXX: Do this properly!
    # Define Python callback function for use in C function. Return a long so
    # it hopefully works on 32 and 64 bit systems.
    allocData = C.CFUNCTYPE(C.c_long, C.c_int, C.c_char)(allocate_data)

    try:
        verbose = int(verbose)
    except:
        verbose = 0

    lil = clibmseed.readMSEEDBuffer(buffer, buflen, selections, unpack_data,
        reclen, C.c_int(verbose), C.c_int(details), header_byteorder,
        allocData)

    # XXX: Check if the freeing works.
    del selections

    traces = []
    try:
        currentID = lil.contents
    # Return stream if not traces are found.
    except ValueError:
        clibmseed.lil_free(lil)
        del lil
        return Stream()

    while True:
        # Init header with the essential information.
        header = {'network': currentID.network.strip(),
                  'station': currentID.station.strip(),
                  'location': currentID.location.strip(),
                  'channel': currentID.channel.strip(),
                  'mseed': {'dataquality': currentID.dataquality}}
        # Loop over segments.
        try:
            currentSegment = currentID.firstSegment.contents
        except ValueError:
            break
        while True:
            header['sampling_rate'] = currentSegment.samprate
            header['starttime'] = \
                util._convertMSTimeToDatetime(currentSegment.starttime)
            # TODO: write support is missing
            if details:
                timing_quality = currentSegment.timing_quality
                if timing_quality == 0xFF:  # 0xFF is mask for not known timing
                    timing_quality = -1
                header['mseed']['timing_quality'] = timing_quality
                header['mseed']['calibration_type'] = \
                        currentSegment.calibration_type

            if headonly is False:
                # The data always will be in sequential order.
                data = all_data.pop(0)
                header['npts'] = len(data)
            else:
                data = np.array([])
                header['npts'] = currentSegment.samplecnt
            # Make sure to init the number of samples.
            trace = Trace(header=header, data=data)
            # Append information if necessary.
            if recinfo:
                for key, value in info.iteritems():
                    setattr(trace.stats.mseed, key, value)
            traces.append(trace)
            # A Null pointer access results in a ValueError
            try:
                currentSegment = currentSegment.next.contents
            except ValueError:
                break
        try:
            currentID = currentID.next.contents
        except ValueError:
            break

    clibmseed.lil_free(lil)
    del lil
    return Stream(traces=traces)