def createIndex(self): import datetime as dt from davitpy.pydarn.dmapio import getDmapOffset,readDmapRec,setDmapOffset recordDict={} starting_offset=self.offsetTell() #rewind back to start of file self.rewind() while(1): #read the next record from the dmap file offset= getDmapOffset(self.__fd) dfile = readDmapRec(self.__fd) if(dfile is None): #if we dont have valid data, clean up, get out print '\nreached end of data' break else: try: dtime = dt.datetime(dfile['start.year'],dfile['start.month'],dfile['start.day'], \ dfile['start.hour'],dfile['start.minute'],int(dfile['start.second'])) dfile['time'] = (dtime - dt.datetime(1970, 1, 1)).total_seconds() except Exception,e: print e print 'problem reading time from file, returning None' break if(dt.datetime.utcfromtimestamp(dfile['time']) >= self.sTime and \ dt.datetime.utcfromtimestamp(dfile['time']) <= self.eTime) : rectime = dt.datetime.utcfromtimestamp(dfile['time']) recordDict[rectime]=offset
def __createIndexDmap(self): """ Create dictionary of offsets as a function of timestamp. """ # This method will have to do different things depending # on self.dType (for future other data file support ie. hdf5) import datetime as dt from davitpy.pydarn.dmapio import getDmapOffset,readDmapRec,setDmapOffset recordDict={} scanStartDict={} starting_offset=self.__offsetTellDmap() #rewind back to start of file self.__rewindDmap() while(1): #read the next record from the dmap file offset= getDmapOffset(self._fd) dfile = readDmapRec(self._fd) if(dfile is None): #if we dont have valid data, clean up, get out print '\nreached end of data' break else: if(dt.datetime.utcfromtimestamp(dfile['time']) >= self.sTime and \ dt.datetime.utcfromtimestamp(dfile['time']) <= self.eTime) : rectime = dt.datetime.utcfromtimestamp(dfile['time']) recordDict[rectime]=offset if dfile['scan']==1: scanStartDict[rectime]=offset #reset back to before building the index self.recordIndex=recordDict self.__offsetSeekDmap(starting_offset) self.scanStartIndex=scanStartDict return recordDict,scanStartDict
def __offsetTellDmap(self): """ Jump to dmap record at supplied byte offset. """ # This method will have to do different things depending # on self.dType (for future other data file support ie. hdf5) from davitpy.pydarn.dmapio import getDmapOffset return getDmapOffset(self._fd)
def offsetSeek(self,offset,force=False): """jump to dmap record at supplied byte offset. Require offset to be in record index list unless forced. """ from davitpy.pydarn.dmapio import setDmapOffset,getDmapOffset if force: return setDmapOffset(self.__fd,offset) else: if self.recordIndex is None: self.createIndex() if offset in self.recordIndex.values(): return setDmapOffset(self.__fd,offset) else: return getDmapOffset(self.__fd)
def __offsetSeekDmap(self, offset, force=False): """ Jump to dmap record at supplied byte offset. Require offset to be in record index list unless forced. """ # This method will have to do different things depending # on self.dType (for future other data file support ie. hdf5) from davitpy.pydarn.dmapio import setDmapOffset, getDmapOffset if force: return setDmapOffset(self._fd, offset) else: if self.recordIndex is None: self.__createIndexDmap() if offset in self.recordIndex.values(): return setDmapOffset(self._fd, offset) else: return getDmapOffset(self._fd)
def __offsetSeekDmap(self,offset,force=False): """ Jump to dmap record at supplied byte offset. Require offset to be in record index list unless forced. """ # This method will have to do different things depending # on self.dType (for future other data file support ie. hdf5) from davitpy.pydarn.dmapio import setDmapOffset,getDmapOffset if force: return setDmapOffset(self._fd,offset) else: if self.recordIndex is None: self.__createIndexDmap() if offset in self.recordIndex.values(): return setDmapOffset(self._fd,offset) else: return getDmapOffset(self._fd)
def createIndex(self): import datetime as dt import davitpy.pydarn.dmapio as dmapio recordDict = {} starting_offset = self.offsetTell() # rewind back to start of file self.rewind() while 1: # read the next record from the dmap file offset = dmapio.getDmapOffset(self.__fd) dfile = dmapio.readDmapRec(self.__fd) if dfile is None: # if we dont have valid data, clean up, get out logging.info('reached end of data') break else: try: dtime = dt.datetime(dfile['start.year'], dfile['start.month'], dfile['start.day'], dfile['start.hour'], dfile['start.minute'], int(dfile['start.second'])) dfile['time'] = (dtime - dt.datetime(1970, 1, 1)).total_seconds() except Exception,e: logging.warning(e) logging.warning('problem reading time from file') break dfile_utc = dt.datetime.utcfromtimestamp(dfile['time']) if dfile_utc >= self.sTime and dfile_utc <= self.eTime: rectime = dt.datetime.utcfromtimestamp(dfile['time']) recordDict[rectime] = offset
def createIndex(self): import datetime as dt import davitpy.pydarn.dmapio as dmapio recordDict = {} starting_offset = self.offsetTell() # rewind back to start of file self.rewind() while 1: # read the next record from the dmap file offset = dmapio.getDmapOffset(self.__fd) dfile = dmapio.readDmapRec(self.__fd) if dfile is None: # if we dont have valid data, clean up, get out logging.info('reached end of data') break else: try: dtime = dt.datetime(dfile['start.year'], dfile['start.month'], dfile['start.day'], dfile['start.hour'], dfile['start.minute'], int(dfile['start.second'])) dfile['time'] = (dtime - dt.datetime(1970, 1, 1)).total_seconds() except Exception, e: logging.warning(e) logging.warning('problem reading time from file') break dfile_utc = dt.datetime.utcfromtimestamp(dfile['time']) if dfile_utc >= self.sTime and dfile_utc <= self.eTime: rectime = dt.datetime.utcfromtimestamp(dfile['time']) recordDict[rectime] = offset
def readRec(self): """A function to read a single record of radar data from a radDataPtr object Returns -------- mydata : (gridData, mapData, or NoneType) An object filled with the specified type of data. Will return None when there is no more data in the pointer to read. """ import davitpy.pydarn.dmapio as dmapio import datetime as dt # check input if self.__ptr == None: logging.error('the pointer does not point to any data') return None if self.__ptr.closed: logging.error('the file pointer is closed') return None # do this until we reach the requested start time # and have a parameter match while 1: offset = dmapio.getDmapOffset(self.__fd) dfile = dmapio.readDmapRec(self.__fd) # check for valid data try: dtime = dt.datetime(dfile['start.year'], dfile['start.month'], dfile['start.day'], dfile['start.hour'], dfile['start.minute'], int(dfile['start.second'])) dfile['time'] = (dtime - dt.datetime(1970, 1, 1)).total_seconds() except Exception, e: logging.warning(e) logging.warning('problem reading time from file') break if(dfile == None or dt.datetime.utcfromtimestamp(dfile['time']) > self.eTime): # if we dont have valid data, clean up, get out logging.info('reached end of data') return None # check that we're in the time window, and that we have a # match for the desired params if(dt.datetime.utcfromtimestamp(dfile['time']) >= self.sTime and dt.datetime.utcfromtimestamp(dfile['time']) <= self.eTime): # fill the beamdata object, checking the file type if self.fType == 'grd' or self.fType == 'grdex': mydata = gridData(dataDict=dfile) elif self.fType == 'map' or self.fType == 'mapex': mydata = mapData(dataDict=dfile) else: logging.error('unrecognized file type') return None mydata.recordDict = dfile mydata.fType = self.fType mydata.fPtr = self mydata.offset = offset return mydata
def offsetTell(self): """jump to dmap record at supplied byte offset. """ from davitpy.pydarn.dmapio import getDmapOffset return getDmapOffset(self.__fd)
def readRec(self): """A function to read a single record of radar data from a radDataPtr object Returns -------- mydata : (gridData, mapData, or NoneType) An object filled with the specified type of data. Will return None when there is no more data in the pointer to read. """ import davitpy.pydarn.dmapio as dmapio import datetime as dt # check input if self.__ptr == None: logging.error('the pointer does not point to any data') return None if self.__ptr.closed: logging.error('the file pointer is closed') return None # do this until we reach the requested start time # and have a parameter match while 1: offset = dmapio.getDmapOffset(self.__fd) dfile = dmapio.readDmapRec(self.__fd) # check for valid data try: dtime = dt.datetime(dfile['start.year'], dfile['start.month'], dfile['start.day'], dfile['start.hour'], dfile['start.minute'], int(dfile['start.second'])) dfile['time'] = (dtime - dt.datetime(1970, 1, 1)).total_seconds() except Exception, e: logging.warning(e) logging.warning('problem reading time from file') break if (dfile == None or dt.datetime.utcfromtimestamp(dfile['time']) > self.eTime): # if we dont have valid data, clean up, get out logging.info('reached end of data') return None # check that we're in the time window, and that we have a # match for the desired params if (dt.datetime.utcfromtimestamp(dfile['time']) >= self.sTime and dt.datetime.utcfromtimestamp(dfile['time']) <= self.eTime): # fill the beamdata object, checking the file type if self.fType == 'grd' or self.fType == 'grdex': mydata = gridData(dataDict=dfile) elif self.fType == 'map' or self.fType == 'mapex': mydata = mapData(dataDict=dfile) else: logging.error('unrecognized file type') return None mydata.recordDict = dfile mydata.fType = self.fType mydata.fPtr = self mydata.offset = offset return mydata