def createIndex(self):
import datetime as dt
from davitpy.pydarn.dmapio import getDmapOffset,readDmapRec,setDmapOffset
recordDict={}
starting_offset=self.offsetTell()
#rewind back to start of file
self.rewind()
while(1):
#read the next record from the dmap file
offset= getDmapOffset(self.__fd)
dfile = readDmapRec(self.__fd)
if(dfile is None):
#if we dont have valid data, clean up, get out
print '\nreached end of data'
break
else:
try:
dtime = dt.datetime(dfile['start.year'],dfile['start.month'],dfile['start.day'], \
dfile['start.hour'],dfile['start.minute'],int(dfile['start.second']))
dfile['time'] = (dtime - dt.datetime(1970, 1, 1)).total_seconds()
except Exception,e:
print e
print 'problem reading time from file, returning None'
break
if(dt.datetime.utcfromtimestamp(dfile['time']) >= self.sTime and \
dt.datetime.utcfromtimestamp(dfile['time']) <= self.eTime) :
rectime = dt.datetime.utcfromtimestamp(dfile['time'])
recordDict[rectime]=offset
def __createIndexDmap(self):
"""
Create dictionary of offsets as a function of timestamp.
"""
# This method will have to do different things depending
# on self.dType (for future other data file support ie. hdf5)
import datetime as dt
from davitpy.pydarn.dmapio import getDmapOffset,readDmapRec,setDmapOffset
recordDict={}
scanStartDict={}
starting_offset=self.__offsetTellDmap()
#rewind back to start of file
self.__rewindDmap()
while(1):
#read the next record from the dmap file
offset= getDmapOffset(self._fd)
dfile = readDmapRec(self._fd)
if(dfile is None):
#if we dont have valid data, clean up, get out
print '\nreached end of data'
break
else:
if(dt.datetime.utcfromtimestamp(dfile['time']) >= self.sTime and \
dt.datetime.utcfromtimestamp(dfile['time']) <= self.eTime) :
rectime = dt.datetime.utcfromtimestamp(dfile['time'])
recordDict[rectime]=offset
if dfile['scan']==1: scanStartDict[rectime]=offset
#reset back to before building the index
self.recordIndex=recordDict
self.__offsetSeekDmap(starting_offset)
self.scanStartIndex=scanStartDict
return recordDict,scanStartDict
def createIndex(self):
import datetime as dt
import davitpy.pydarn.dmapio as dmapio
recordDict = {}
starting_offset = self.offsetTell()
# rewind back to start of file
self.rewind()
while 1:
# read the next record from the dmap file
offset = dmapio.getDmapOffset(self.__fd)
dfile = dmapio.readDmapRec(self.__fd)
if dfile is None:
# if we dont have valid data, clean up, get out
logging.info('reached end of data')
break
else:
try:
dtime = dt.datetime(dfile['start.year'],
dfile['start.month'],
dfile['start.day'],
dfile['start.hour'],
dfile['start.minute'],
int(dfile['start.second']))
dfile['time'] = (dtime -
dt.datetime(1970, 1, 1)).total_seconds()
except Exception,e:
logging.warning(e)
logging.warning('problem reading time from file')
break
dfile_utc = dt.datetime.utcfromtimestamp(dfile['time'])
if dfile_utc >= self.sTime and dfile_utc <= self.eTime:
rectime = dt.datetime.utcfromtimestamp(dfile['time'])
recordDict[rectime] = offset
def createIndex(self):
import datetime as dt
import davitpy.pydarn.dmapio as dmapio
recordDict = {}
starting_offset = self.offsetTell()
# rewind back to start of file
self.rewind()
while 1:
# read the next record from the dmap file
offset = dmapio.getDmapOffset(self.__fd)
dfile = dmapio.readDmapRec(self.__fd)
if dfile is None:
# if we dont have valid data, clean up, get out
logging.info('reached end of data')
break
else:
try:
dtime = dt.datetime(dfile['start.year'],
dfile['start.month'],
dfile['start.day'],
dfile['start.hour'],
dfile['start.minute'],
int(dfile['start.second']))
dfile['time'] = (dtime -
dt.datetime(1970, 1, 1)).total_seconds()
except Exception, e:
logging.warning(e)
logging.warning('problem reading time from file')
break
dfile_utc = dt.datetime.utcfromtimestamp(dfile['time'])
if dfile_utc >= self.sTime and dfile_utc <= self.eTime:
rectime = dt.datetime.utcfromtimestamp(dfile['time'])
recordDict[rectime] = offset
def __validate_fetched(self, filelist, stime, etime):
""" This function checks if the files in filelist contain data
for the start and end times (stime,etime) requested by a user.
Parameters
-----------
filelist : (list)
List of filenames to validate
stime : (datetime.datetime)
Starting time for files
etime : (datetime.datetime)
Ending time for files
Returns
--------
valid : (list of bool)
List of booleans corresponding to each filename. True if a file
contains data in the time range (stime,etime), False if not
"""
# This method will need some modification for it to work with
# file formats that are NOT DMAP (i.e. HDF5). Namely, the dmapio
# specific code will need to be modified (readDmapRec).
import datetime as dt
import numpy as np
from davitpy.pydarn.dmapio import readDmapRec
valid = []
for f in filelist:
logging.info('Checking file: {:s}'.format(f))
stimes = []
etimes = []
# Open the file and create a file pointer
self.__filename = f
self.open()
# Iterate through the file and grab the start time for beam
# integration and calculate the end time from intt.sc and intt.us
while 1:
# read the next record from the dmap file
dfile = readDmapRec(self.__fd)
if(dfile is None):
break
else:
temp = dt.datetime(int(dfile['start.year']),
int(dfile['start.month']),
int(dfile['start.day']),
int(dfile['start.hour']),
int(dfile['start.minute']),
int(dfile['start.second']))
stimes.append(temp)
temp = dt.datetime(int(dfile['end.year']),
int(dfile['end.month']),
int(dfile['end.day']),
int(dfile['end.hour']),
int(dfile['end.minute']),
int(dfile['end.second']))
etimes.append(temp)
# Close the file and clean up
self.close()
self.__ptr = None
inds = np.where((np.array(stimes) >= stime) &
(np.array(stimes) <= etime))
inde = np.where((np.array(etimes) >= stime) &
(np.array(etimes) <= etime))
if np.size(inds) > 0 or np.size(inde) > 0:
valid.append(True)
else:
valid.append(False) # ISSUE 217: FASTER TO NOT USE APPEND
return valid
def readRec(self):
"""A function to read a single record of radar data from a radDataPtr
object
Returns
--------
mydata : (gridData, mapData, or NoneType)
An object filled with the specified type of data. Will return None
when there is no more data in the pointer to read.
"""
import davitpy.pydarn.dmapio as dmapio
import datetime as dt
# check input
if self.__ptr == None:
logging.error('the pointer does not point to any data')
return None
if self.__ptr.closed:
logging.error('the file pointer is closed')
return None
# do this until we reach the requested start time
# and have a parameter match
while 1:
offset = dmapio.getDmapOffset(self.__fd)
dfile = dmapio.readDmapRec(self.__fd)
# check for valid data
try:
dtime = dt.datetime(dfile['start.year'], dfile['start.month'],
dfile['start.day'], dfile['start.hour'],
dfile['start.minute'],
int(dfile['start.second']))
dfile['time'] = (dtime -
dt.datetime(1970, 1, 1)).total_seconds()
except Exception, e:
logging.warning(e)
logging.warning('problem reading time from file')
break
if(dfile == None or
dt.datetime.utcfromtimestamp(dfile['time']) > self.eTime):
# if we dont have valid data, clean up, get out
logging.info('reached end of data')
return None
# check that we're in the time window, and that we have a
# match for the desired params
if(dt.datetime.utcfromtimestamp(dfile['time']) >= self.sTime and
dt.datetime.utcfromtimestamp(dfile['time']) <= self.eTime):
# fill the beamdata object, checking the file type
if self.fType == 'grd' or self.fType == 'grdex':
mydata = gridData(dataDict=dfile)
elif self.fType == 'map' or self.fType == 'mapex':
mydata = mapData(dataDict=dfile)
else:
logging.error('unrecognized file type')
return None
mydata.recordDict = dfile
mydata.fType = self.fType
mydata.fPtr = self
mydata.offset = offset
return mydata
def __validate_fetched(self,filelist,stime,etime):
""" This function checks if the files in filelist contain data
for the start and end times (stime,etime) requested by a user.
**Args**:
* **filelist** (list):
* **stime** (datetime.datetime):
* **etime** (datetime.datetime):
**Returns**:
* List of booleans. True if a file contains data in the time
range (stime,etime)
"""
# This method will need some modification for it to work with
# file formats that are NOT DMAP (i.e. HDF5). Namely, the dmapio
# specific code will need to be modified (readDmapRec).
import datetime as dt
import numpy as np
from davitpy.pydarn.dmapio import readDmapRec
valid = []
for f in filelist:
print 'Checking file: ' + f
stimes = []
etimes = []
# Open the file and create a file pointer
self.__filename = f
self.open()
# Iterate through the file and grab the start time for beam
# integration and calculate the end time from intt.sc and intt.us
while(1):
#read the next record from the dmap file
dfile = readDmapRec(self.__fd)
if(dfile is None):
break
else:
temp = dt.datetime(int(dfile['start.year']),int(dfile['start.month']),
int(dfile['start.day']),int(dfile['start.hour']),
int(dfile['start.minute']),int(dfile['start.second']))
stimes.append(temp)
temp = dt.datetime(int(dfile['end.year']),int(dfile['end.month']),
int(dfile['end.day']),int(dfile['end.hour']),
int(dfile['end.minute']),int(dfile['end.second']))
etimes.append(temp)
# Close the file and clean up
self.close()
self.__ptr = None
inds = np.where((np.array(stimes) >= stime) & (np.array(stimes) <= etime))
inde = np.where((np.array(etimes) >= stime) & (np.array(etimes) <= etime))
if (np.size(inds) > 0) or (np.size(inde) > 0):
valid.append(True)
else:
valid.append(False)
return valid
def __validate_fetched(self, filelist, stime, etime):
""" This function checks if the files in filelist contain data
for the start and end times (stime,etime) requested by a user.
Parameters
-----------
filelist : (list)
List of filenames to validate
stime : (datetime.datetime)
Starting time for files
etime : (datetime.datetime)
Ending time for files
Returns
--------
valid : (list of bool)
List of booleans corresponding to each filename. True if a file
contains data in the time range (stime,etime), False if not
"""
# This method will need some modification for it to work with
# file formats that are NOT DMAP (i.e. HDF5). Namely, the dmapio
# specific code will need to be modified (readDmapRec).
import datetime as dt
import numpy as np
from davitpy.pydarn.dmapio import readDmapRec
valid = []
for f in filelist:
logging.info('Checking file: {:s}'.format(f))
stimes = []
etimes = []
# Open the file and create a file pointer
self.__filename = f
self.open()
# Iterate through the file and grab the start time for beam
# integration and calculate the end time from intt.sc and intt.us
while 1:
# read the next record from the dmap file
dfile = readDmapRec(self.__fd)
if (dfile is None):
break
else:
temp = dt.datetime(int(dfile['start.year']),
int(dfile['start.month']),
int(dfile['start.day']),
int(dfile['start.hour']),
int(dfile['start.minute']),
int(dfile['start.second']))
stimes.append(temp)
temp = dt.datetime(int(dfile['end.year']),
int(dfile['end.month']),
int(dfile['end.day']),
int(dfile['end.hour']),
int(dfile['end.minute']),
int(dfile['end.second']))
etimes.append(temp)
# Close the file and clean up
self.close()
self.__ptr = None
inds = np.where((np.array(stimes) >= stime)
& (np.array(stimes) <= etime))
inde = np.where((np.array(etimes) >= stime)
& (np.array(etimes) <= etime))
if np.size(inds) > 0 or np.size(inde) > 0:
valid.append(True)
else:
valid.append(False) # ISSUE 217: FASTER TO NOT USE APPEND
return valid
def readRec(self):
"""A function to read a single record of radar data from a radDataPtr
object
Returns
--------
mydata : (gridData, mapData, or NoneType)
An object filled with the specified type of data. Will return None
when there is no more data in the pointer to read.
"""
import davitpy.pydarn.dmapio as dmapio
import datetime as dt
# check input
if self.__ptr == None:
logging.error('the pointer does not point to any data')
return None
if self.__ptr.closed:
logging.error('the file pointer is closed')
return None
# do this until we reach the requested start time
# and have a parameter match
while 1:
offset = dmapio.getDmapOffset(self.__fd)
dfile = dmapio.readDmapRec(self.__fd)
# check for valid data
try:
dtime = dt.datetime(dfile['start.year'], dfile['start.month'],
dfile['start.day'], dfile['start.hour'],
dfile['start.minute'],
int(dfile['start.second']))
dfile['time'] = (dtime -
dt.datetime(1970, 1, 1)).total_seconds()
except Exception, e:
logging.warning(e)
logging.warning('problem reading time from file')
break
if (dfile == None or
dt.datetime.utcfromtimestamp(dfile['time']) > self.eTime):
# if we dont have valid data, clean up, get out
logging.info('reached end of data')
return None
# check that we're in the time window, and that we have a
# match for the desired params
if (dt.datetime.utcfromtimestamp(dfile['time']) >= self.sTime and
dt.datetime.utcfromtimestamp(dfile['time']) <= self.eTime):
# fill the beamdata object, checking the file type
if self.fType == 'grd' or self.fType == 'grdex':
mydata = gridData(dataDict=dfile)
elif self.fType == 'map' or self.fType == 'mapex':
mydata = mapData(dataDict=dfile)
else:
logging.error('unrecognized file type')
return None
mydata.recordDict = dfile
mydata.fType = self.fType
mydata.fPtr = self
mydata.offset = offset
return mydata