def createIndex(self):
        import datetime as dt
        from davitpy.pydarn.dmapio import getDmapOffset,readDmapRec,setDmapOffset
        recordDict={}
        starting_offset=self.offsetTell()
        #rewind back to start of file
        self.rewind()
        while(1):
            #read the next record from the dmap file
            offset= getDmapOffset(self.__fd)
            dfile = readDmapRec(self.__fd)
            if(dfile is None):
                #if we dont have valid data, clean up, get out
                print '\nreached end of data'
                break
            else:
                try:
                    dtime = dt.datetime(dfile['start.year'],dfile['start.month'],dfile['start.day'], \
                                 dfile['start.hour'],dfile['start.minute'],int(dfile['start.second']))
                    dfile['time'] = (dtime - dt.datetime(1970, 1, 1)).total_seconds()
                except Exception,e:
                    print e
                    print 'problem reading time from file, returning None'
                    break

                if(dt.datetime.utcfromtimestamp(dfile['time']) >= self.sTime and \
                  dt.datetime.utcfromtimestamp(dfile['time']) <= self.eTime) : 
                    rectime = dt.datetime.utcfromtimestamp(dfile['time'])
                    recordDict[rectime]=offset
Exemple #2
0
    def __createIndexDmap(self):
        """
           Create dictionary of offsets as a function of timestamp.
  
        """

        # This method will have to do different things depending 
        # on self.dType (for future other data file support ie. hdf5)

        import datetime as dt
        from davitpy.pydarn.dmapio import getDmapOffset,readDmapRec,setDmapOffset
        recordDict={}
        scanStartDict={}
        starting_offset=self.__offsetTellDmap()
        #rewind back to start of file
        self.__rewindDmap()
        while(1):
            #read the next record from the dmap file
            offset= getDmapOffset(self._fd)
            dfile = readDmapRec(self._fd)
            if(dfile is None):
                #if we dont have valid data, clean up, get out
                print '\nreached end of data'
                break
            else:
                if(dt.datetime.utcfromtimestamp(dfile['time']) >= self.sTime and \
                  dt.datetime.utcfromtimestamp(dfile['time']) <= self.eTime) : 
                    rectime = dt.datetime.utcfromtimestamp(dfile['time'])
                    recordDict[rectime]=offset
                    if dfile['scan']==1: scanStartDict[rectime]=offset
        #reset back to before building the index 
        self.recordIndex=recordDict
        self.__offsetSeekDmap(starting_offset)
        self.scanStartIndex=scanStartDict
        return recordDict,scanStartDict
Exemple #3
0
    def createIndex(self):
        import datetime as dt
        import davitpy.pydarn.dmapio as dmapio

        recordDict = {}
        starting_offset = self.offsetTell()

        # rewind back to start of file
        self.rewind()
        while 1:
            # read the next record from the dmap file
            offset = dmapio.getDmapOffset(self.__fd)
            dfile = dmapio.readDmapRec(self.__fd)
            if dfile is None:
                # if we dont have valid data, clean up, get out
                logging.info('reached end of data')
                break
            else:
                try:
                    dtime = dt.datetime(dfile['start.year'],
                                        dfile['start.month'],
                                        dfile['start.day'],
                                        dfile['start.hour'],
                                        dfile['start.minute'],
                                        int(dfile['start.second']))
                    dfile['time'] = (dtime -
                                     dt.datetime(1970, 1, 1)).total_seconds()
                except Exception,e:
                    logging.warning(e)
                    logging.warning('problem reading time from file')
                    break

                dfile_utc = dt.datetime.utcfromtimestamp(dfile['time'])
                if dfile_utc >= self.sTime and dfile_utc <= self.eTime: 
                    rectime = dt.datetime.utcfromtimestamp(dfile['time'])
                    recordDict[rectime] = offset
Exemple #4
0
    def createIndex(self):
        import datetime as dt
        import davitpy.pydarn.dmapio as dmapio

        recordDict = {}
        starting_offset = self.offsetTell()

        # rewind back to start of file
        self.rewind()
        while 1:
            # read the next record from the dmap file
            offset = dmapio.getDmapOffset(self.__fd)
            dfile = dmapio.readDmapRec(self.__fd)
            if dfile is None:
                # if we dont have valid data, clean up, get out
                logging.info('reached end of data')
                break
            else:
                try:
                    dtime = dt.datetime(dfile['start.year'],
                                        dfile['start.month'],
                                        dfile['start.day'],
                                        dfile['start.hour'],
                                        dfile['start.minute'],
                                        int(dfile['start.second']))
                    dfile['time'] = (dtime -
                                     dt.datetime(1970, 1, 1)).total_seconds()
                except Exception, e:
                    logging.warning(e)
                    logging.warning('problem reading time from file')
                    break

                dfile_utc = dt.datetime.utcfromtimestamp(dfile['time'])
                if dfile_utc >= self.sTime and dfile_utc <= self.eTime:
                    rectime = dt.datetime.utcfromtimestamp(dfile['time'])
                    recordDict[rectime] = offset
Exemple #5
0
    def __validate_fetched(self, filelist, stime, etime):
        """ This function checks if the files in filelist contain data
        for the start and end times (stime,etime) requested by a user.

        Parameters
        -----------
        filelist : (list)
            List of filenames to validate
        stime : (datetime.datetime)
            Starting time for files
        etime : (datetime.datetime)
            Ending time for files

        Returns
        --------
        valid : (list of bool)
            List of booleans corresponding to each filename. True if a file
            contains data in the time range (stime,etime), False if not
        """
        # This method will need some modification for it to work with
        # file formats that are NOT DMAP (i.e. HDF5). Namely, the dmapio
        # specific code will need to be modified (readDmapRec).

        import datetime as dt
        import numpy as np
        from davitpy.pydarn.dmapio import readDmapRec

        valid = []

        for f in filelist:
            logging.info('Checking file: {:s}'.format(f))
            stimes = []
            etimes = []

            # Open the file and create a file pointer
            self.__filename = f
            self.open()

            # Iterate through the file and grab the start time for beam
            # integration and calculate the end time from intt.sc and intt.us
            while 1:
                # read the next record from the dmap file
                dfile = readDmapRec(self.__fd)
                if(dfile is None):
                    break
                else:
                    temp = dt.datetime(int(dfile['start.year']),
                                       int(dfile['start.month']),
                                       int(dfile['start.day']),
                                       int(dfile['start.hour']),
                                       int(dfile['start.minute']),
                                       int(dfile['start.second']))
                    stimes.append(temp)
                    temp = dt.datetime(int(dfile['end.year']),
                                       int(dfile['end.month']),
                                       int(dfile['end.day']),
                                       int(dfile['end.hour']),
                                       int(dfile['end.minute']),
                                       int(dfile['end.second']))
                    etimes.append(temp)

            # Close the file and clean up
            self.close()
            self.__ptr = None

            inds = np.where((np.array(stimes) >= stime) &
                            (np.array(stimes) <= etime))
            inde = np.where((np.array(etimes) >= stime) &
                            (np.array(etimes) <= etime))

            if np.size(inds) > 0 or np.size(inde) > 0:
                valid.append(True)
            else:
                valid.append(False) # ISSUE 217: FASTER TO NOT USE APPEND

        return valid
Exemple #6
0
    def readRec(self):
        """A function to read a single record of radar data from a radDataPtr
        object

        Returns
        --------
        mydata : (gridData, mapData, or NoneType)
            An object filled with the specified type of data.  Will return None
            when there is no more data in the pointer to read.
        """
        import davitpy.pydarn.dmapio as dmapio
        import datetime as dt

        # check input
        if self.__ptr == None:
            logging.error('the pointer does not point to any data')
            return None

        if self.__ptr.closed:
            logging.error('the file pointer is closed')
            return None
  
        # do this until we reach the requested start time
        # and have a parameter match
        while 1:
            offset = dmapio.getDmapOffset(self.__fd)
            dfile = dmapio.readDmapRec(self.__fd)
            # check for valid data
            try:
                dtime = dt.datetime(dfile['start.year'], dfile['start.month'],
                                    dfile['start.day'], dfile['start.hour'],
                                    dfile['start.minute'],
                                    int(dfile['start.second']))
                dfile['time'] = (dtime -
                                 dt.datetime(1970, 1, 1)).total_seconds()

            except Exception, e:
                logging.warning(e)
                logging.warning('problem reading time from file')
                break

            if(dfile == None or
               dt.datetime.utcfromtimestamp(dfile['time']) > self.eTime):
                # if we dont have valid data, clean up, get out
                logging.info('reached end of data')
                return None

            # check that we're in the time window, and that we have a 
            # match for the desired params  
            if(dt.datetime.utcfromtimestamp(dfile['time']) >= self.sTime and
               dt.datetime.utcfromtimestamp(dfile['time']) <= self.eTime):
                # fill the beamdata object, checking the file type
                if self.fType == 'grd' or self.fType == 'grdex':
                    mydata = gridData(dataDict=dfile)
                elif self.fType == 'map' or self.fType == 'mapex':
                    mydata = mapData(dataDict=dfile)
                else:
                    logging.error('unrecognized file type')
                    return None

                mydata.recordDict = dfile
                mydata.fType = self.fType
                mydata.fPtr = self
                mydata.offset = offset
  
                return mydata
    def __validate_fetched(self,filelist,stime,etime):
        """ This function checks if the files in filelist contain data
        for the start and end times (stime,etime) requested by a user.

        **Args**:
            * **filelist** (list):
            * **stime** (datetime.datetime):
            * **etime** (datetime.datetime):

        **Returns**:
            * List of booleans. True if a file contains data in the time
            range (stime,etime)
        """

        # This method will need some modification for it to work with
        # file formats that are NOT DMAP (i.e. HDF5). Namely, the dmapio
        # specific code will need to be modified (readDmapRec).

        import datetime as dt
        import numpy as np
        from davitpy.pydarn.dmapio import readDmapRec

        valid = []

        for f in filelist:
            print 'Checking file: ' + f
            stimes = []
            etimes = []

            # Open the file and create a file pointer
            self.__filename = f
            self.open()

            # Iterate through the file and grab the start time for beam
            # integration and calculate the end time from intt.sc and intt.us
            while(1):
                #read the next record from the dmap file
                dfile = readDmapRec(self.__fd)
                if(dfile is None):
                    break
                else:
                    temp = dt.datetime(int(dfile['start.year']),int(dfile['start.month']),
                                       int(dfile['start.day']),int(dfile['start.hour']),
                                       int(dfile['start.minute']),int(dfile['start.second']))
                    stimes.append(temp)
                    temp = dt.datetime(int(dfile['end.year']),int(dfile['end.month']),
                                       int(dfile['end.day']),int(dfile['end.hour']),
                                       int(dfile['end.minute']),int(dfile['end.second']))
                    etimes.append(temp)
            # Close the file and clean up
            self.close()
            self.__ptr = None

            inds = np.where((np.array(stimes) >= stime) & (np.array(stimes) <= etime))
            inde = np.where((np.array(etimes) >= stime) & (np.array(etimes) <= etime))
            if (np.size(inds) > 0) or (np.size(inde) > 0):
                valid.append(True)
            else:
                valid.append(False)

        return valid
Exemple #8
0
    def __validate_fetched(self, filelist, stime, etime):
        """ This function checks if the files in filelist contain data
        for the start and end times (stime,etime) requested by a user.

        Parameters
        -----------
        filelist : (list)
            List of filenames to validate
        stime : (datetime.datetime)
            Starting time for files
        etime : (datetime.datetime)
            Ending time for files

        Returns
        --------
        valid : (list of bool)
            List of booleans corresponding to each filename. True if a file
            contains data in the time range (stime,etime), False if not
        """
        # This method will need some modification for it to work with
        # file formats that are NOT DMAP (i.e. HDF5). Namely, the dmapio
        # specific code will need to be modified (readDmapRec).

        import datetime as dt
        import numpy as np
        from davitpy.pydarn.dmapio import readDmapRec

        valid = []

        for f in filelist:
            logging.info('Checking file: {:s}'.format(f))
            stimes = []
            etimes = []

            # Open the file and create a file pointer
            self.__filename = f
            self.open()

            # Iterate through the file and grab the start time for beam
            # integration and calculate the end time from intt.sc and intt.us
            while 1:
                # read the next record from the dmap file
                dfile = readDmapRec(self.__fd)
                if (dfile is None):
                    break
                else:
                    temp = dt.datetime(int(dfile['start.year']),
                                       int(dfile['start.month']),
                                       int(dfile['start.day']),
                                       int(dfile['start.hour']),
                                       int(dfile['start.minute']),
                                       int(dfile['start.second']))
                    stimes.append(temp)
                    temp = dt.datetime(int(dfile['end.year']),
                                       int(dfile['end.month']),
                                       int(dfile['end.day']),
                                       int(dfile['end.hour']),
                                       int(dfile['end.minute']),
                                       int(dfile['end.second']))
                    etimes.append(temp)

            # Close the file and clean up
            self.close()
            self.__ptr = None

            inds = np.where((np.array(stimes) >= stime)
                            & (np.array(stimes) <= etime))
            inde = np.where((np.array(etimes) >= stime)
                            & (np.array(etimes) <= etime))

            if np.size(inds) > 0 or np.size(inde) > 0:
                valid.append(True)
            else:
                valid.append(False)  # ISSUE 217: FASTER TO NOT USE APPEND

        return valid
Exemple #9
0
    def readRec(self):
        """A function to read a single record of radar data from a radDataPtr
        object

        Returns
        --------
        mydata : (gridData, mapData, or NoneType)
            An object filled with the specified type of data.  Will return None
            when there is no more data in the pointer to read.
        """
        import davitpy.pydarn.dmapio as dmapio
        import datetime as dt

        # check input
        if self.__ptr == None:
            logging.error('the pointer does not point to any data')
            return None

        if self.__ptr.closed:
            logging.error('the file pointer is closed')
            return None

        # do this until we reach the requested start time
        # and have a parameter match
        while 1:
            offset = dmapio.getDmapOffset(self.__fd)
            dfile = dmapio.readDmapRec(self.__fd)
            # check for valid data
            try:
                dtime = dt.datetime(dfile['start.year'], dfile['start.month'],
                                    dfile['start.day'], dfile['start.hour'],
                                    dfile['start.minute'],
                                    int(dfile['start.second']))
                dfile['time'] = (dtime -
                                 dt.datetime(1970, 1, 1)).total_seconds()

            except Exception, e:
                logging.warning(e)
                logging.warning('problem reading time from file')
                break

            if (dfile == None or
                    dt.datetime.utcfromtimestamp(dfile['time']) > self.eTime):
                # if we dont have valid data, clean up, get out
                logging.info('reached end of data')
                return None

            # check that we're in the time window, and that we have a
            # match for the desired params
            if (dt.datetime.utcfromtimestamp(dfile['time']) >= self.sTime and
                    dt.datetime.utcfromtimestamp(dfile['time']) <= self.eTime):
                # fill the beamdata object, checking the file type
                if self.fType == 'grd' or self.fType == 'grdex':
                    mydata = gridData(dataDict=dfile)
                elif self.fType == 'map' or self.fType == 'mapex':
                    mydata = mapData(dataDict=dfile)
                else:
                    logging.error('unrecognized file type')
                    return None

                mydata.recordDict = dfile
                mydata.fType = self.fType
                mydata.fPtr = self
                mydata.offset = offset

                return mydata