def createIndex(self):
import datetime as dt
from davitpy.pydarn.dmapio import getDmapOffset,readDmapRec,setDmapOffset
recordDict={}
starting_offset=self.offsetTell()
#rewind back to start of file
self.rewind()
while(1):
#read the next record from the dmap file
offset= getDmapOffset(self.__fd)
dfile = readDmapRec(self.__fd)
if(dfile is None):
#if we dont have valid data, clean up, get out
print '\nreached end of data'
break
else:
try:
dtime = dt.datetime(dfile['start.year'],dfile['start.month'],dfile['start.day'], \
dfile['start.hour'],dfile['start.minute'],int(dfile['start.second']))
dfile['time'] = (dtime - dt.datetime(1970, 1, 1)).total_seconds()
except Exception,e:
print e
print 'problem reading time from file, returning None'
break
if(dt.datetime.utcfromtimestamp(dfile['time']) >= self.sTime and \
dt.datetime.utcfromtimestamp(dfile['time']) <= self.eTime) :
rectime = dt.datetime.utcfromtimestamp(dfile['time'])
recordDict[rectime]=offset
def __createIndexDmap(self):
"""
Create dictionary of offsets as a function of timestamp.
"""
# This method will have to do different things depending
# on self.dType (for future other data file support ie. hdf5)
import datetime as dt
from davitpy.pydarn.dmapio import getDmapOffset,readDmapRec,setDmapOffset
recordDict={}
scanStartDict={}
starting_offset=self.__offsetTellDmap()
#rewind back to start of file
self.__rewindDmap()
while(1):
#read the next record from the dmap file
offset= getDmapOffset(self._fd)
dfile = readDmapRec(self._fd)
if(dfile is None):
#if we dont have valid data, clean up, get out
print '\nreached end of data'
break
else:
if(dt.datetime.utcfromtimestamp(dfile['time']) >= self.sTime and \
dt.datetime.utcfromtimestamp(dfile['time']) <= self.eTime) :
rectime = dt.datetime.utcfromtimestamp(dfile['time'])
recordDict[rectime]=offset
if dfile['scan']==1: scanStartDict[rectime]=offset
#reset back to before building the index
self.recordIndex=recordDict
self.__offsetSeekDmap(starting_offset)
self.scanStartIndex=scanStartDict
return recordDict,scanStartDict
def __offsetTellDmap(self):
""" Jump to dmap record at supplied byte offset.
"""
# This method will have to do different things depending
# on self.dType (for future other data file support ie. hdf5)
from davitpy.pydarn.dmapio import getDmapOffset
return getDmapOffset(self._fd)
def __offsetTellDmap(self):
""" Jump to dmap record at supplied byte offset.
"""
# This method will have to do different things depending
# on self.dType (for future other data file support ie. hdf5)
from davitpy.pydarn.dmapio import getDmapOffset
return getDmapOffset(self._fd)
def offsetSeek(self,offset,force=False):
"""jump to dmap record at supplied byte offset.
Require offset to be in record index list unless forced.
"""
from davitpy.pydarn.dmapio import setDmapOffset,getDmapOffset
if force:
return setDmapOffset(self.__fd,offset)
else:
if self.recordIndex is None:
self.createIndex()
if offset in self.recordIndex.values():
return setDmapOffset(self.__fd,offset)
else:
return getDmapOffset(self.__fd)
def __offsetSeekDmap(self, offset, force=False):
""" Jump to dmap record at supplied byte offset.
Require offset to be in record index list unless forced.
"""
# This method will have to do different things depending
# on self.dType (for future other data file support ie. hdf5)
from davitpy.pydarn.dmapio import setDmapOffset, getDmapOffset
if force:
return setDmapOffset(self._fd, offset)
else:
if self.recordIndex is None:
self.__createIndexDmap()
if offset in self.recordIndex.values():
return setDmapOffset(self._fd, offset)
else:
return getDmapOffset(self._fd)
def __offsetSeekDmap(self,offset,force=False):
"""
Jump to dmap record at supplied byte offset.
Require offset to be in record index list unless forced.
"""
# This method will have to do different things depending
# on self.dType (for future other data file support ie. hdf5)
from davitpy.pydarn.dmapio import setDmapOffset,getDmapOffset
if force:
return setDmapOffset(self._fd,offset)
else:
if self.recordIndex is None:
self.__createIndexDmap()
if offset in self.recordIndex.values():
return setDmapOffset(self._fd,offset)
else:
return getDmapOffset(self._fd)
def createIndex(self):
import datetime as dt
import davitpy.pydarn.dmapio as dmapio
recordDict = {}
starting_offset = self.offsetTell()
# rewind back to start of file
self.rewind()
while 1:
# read the next record from the dmap file
offset = dmapio.getDmapOffset(self.__fd)
dfile = dmapio.readDmapRec(self.__fd)
if dfile is None:
# if we dont have valid data, clean up, get out
logging.info('reached end of data')
break
else:
try:
dtime = dt.datetime(dfile['start.year'],
dfile['start.month'],
dfile['start.day'],
dfile['start.hour'],
dfile['start.minute'],
int(dfile['start.second']))
dfile['time'] = (dtime -
dt.datetime(1970, 1, 1)).total_seconds()
except Exception,e:
logging.warning(e)
logging.warning('problem reading time from file')
break
dfile_utc = dt.datetime.utcfromtimestamp(dfile['time'])
if dfile_utc >= self.sTime and dfile_utc <= self.eTime:
rectime = dt.datetime.utcfromtimestamp(dfile['time'])
recordDict[rectime] = offset
def createIndex(self):
import datetime as dt
import davitpy.pydarn.dmapio as dmapio
recordDict = {}
starting_offset = self.offsetTell()
# rewind back to start of file
self.rewind()
while 1:
# read the next record from the dmap file
offset = dmapio.getDmapOffset(self.__fd)
dfile = dmapio.readDmapRec(self.__fd)
if dfile is None:
# if we dont have valid data, clean up, get out
logging.info('reached end of data')
break
else:
try:
dtime = dt.datetime(dfile['start.year'],
dfile['start.month'],
dfile['start.day'],
dfile['start.hour'],
dfile['start.minute'],
int(dfile['start.second']))
dfile['time'] = (dtime -
dt.datetime(1970, 1, 1)).total_seconds()
except Exception, e:
logging.warning(e)
logging.warning('problem reading time from file')
break
dfile_utc = dt.datetime.utcfromtimestamp(dfile['time'])
if dfile_utc >= self.sTime and dfile_utc <= self.eTime:
rectime = dt.datetime.utcfromtimestamp(dfile['time'])
recordDict[rectime] = offset
def readRec(self):
"""A function to read a single record of radar data from a radDataPtr
object
Returns
--------
mydata : (gridData, mapData, or NoneType)
An object filled with the specified type of data. Will return None
when there is no more data in the pointer to read.
"""
import davitpy.pydarn.dmapio as dmapio
import datetime as dt
# check input
if self.__ptr == None:
logging.error('the pointer does not point to any data')
return None
if self.__ptr.closed:
logging.error('the file pointer is closed')
return None
# do this until we reach the requested start time
# and have a parameter match
while 1:
offset = dmapio.getDmapOffset(self.__fd)
dfile = dmapio.readDmapRec(self.__fd)
# check for valid data
try:
dtime = dt.datetime(dfile['start.year'], dfile['start.month'],
dfile['start.day'], dfile['start.hour'],
dfile['start.minute'],
int(dfile['start.second']))
dfile['time'] = (dtime -
dt.datetime(1970, 1, 1)).total_seconds()
except Exception, e:
logging.warning(e)
logging.warning('problem reading time from file')
break
if(dfile == None or
dt.datetime.utcfromtimestamp(dfile['time']) > self.eTime):
# if we dont have valid data, clean up, get out
logging.info('reached end of data')
return None
# check that we're in the time window, and that we have a
# match for the desired params
if(dt.datetime.utcfromtimestamp(dfile['time']) >= self.sTime and
dt.datetime.utcfromtimestamp(dfile['time']) <= self.eTime):
# fill the beamdata object, checking the file type
if self.fType == 'grd' or self.fType == 'grdex':
mydata = gridData(dataDict=dfile)
elif self.fType == 'map' or self.fType == 'mapex':
mydata = mapData(dataDict=dfile)
else:
logging.error('unrecognized file type')
return None
mydata.recordDict = dfile
mydata.fType = self.fType
mydata.fPtr = self
mydata.offset = offset
return mydata
def offsetTell(self):
"""jump to dmap record at supplied byte offset.
"""
from davitpy.pydarn.dmapio import getDmapOffset
return getDmapOffset(self.__fd)
def offsetTell(self):
"""jump to dmap record at supplied byte offset.
"""
from davitpy.pydarn.dmapio import getDmapOffset
return getDmapOffset(self.__fd)
def readRec(self):
"""A function to read a single record of radar data from a radDataPtr
object
Returns
--------
mydata : (gridData, mapData, or NoneType)
An object filled with the specified type of data. Will return None
when there is no more data in the pointer to read.
"""
import davitpy.pydarn.dmapio as dmapio
import datetime as dt
# check input
if self.__ptr == None:
logging.error('the pointer does not point to any data')
return None
if self.__ptr.closed:
logging.error('the file pointer is closed')
return None
# do this until we reach the requested start time
# and have a parameter match
while 1:
offset = dmapio.getDmapOffset(self.__fd)
dfile = dmapio.readDmapRec(self.__fd)
# check for valid data
try:
dtime = dt.datetime(dfile['start.year'], dfile['start.month'],
dfile['start.day'], dfile['start.hour'],
dfile['start.minute'],
int(dfile['start.second']))
dfile['time'] = (dtime -
dt.datetime(1970, 1, 1)).total_seconds()
except Exception, e:
logging.warning(e)
logging.warning('problem reading time from file')
break
if (dfile == None or
dt.datetime.utcfromtimestamp(dfile['time']) > self.eTime):
# if we dont have valid data, clean up, get out
logging.info('reached end of data')
return None
# check that we're in the time window, and that we have a
# match for the desired params
if (dt.datetime.utcfromtimestamp(dfile['time']) >= self.sTime and
dt.datetime.utcfromtimestamp(dfile['time']) <= self.eTime):
# fill the beamdata object, checking the file type
if self.fType == 'grd' or self.fType == 'grdex':
mydata = gridData(dataDict=dfile)
elif self.fType == 'map' or self.fType == 'mapex':
mydata = mapData(dataDict=dfile)
else:
logging.error('unrecognized file type')
return None
mydata.recordDict = dfile
mydata.fType = self.fType
mydata.fPtr = self
mydata.offset = offset
return mydata
