def __init__(self,sTime=None,radcode=None,eTime=None,stid=None,channel=None,bmnum=None,cp=None, \
fileType=None,filtered=False, src=None,fileName=None,noCache=False):
import datetime as dt
import os,glob,string
from pydarn.radar import network
self.sTime = sTime
self.eTime = eTime
self.stid = stid
self.channel = channel
self.bmnum = bmnum
self.cp = cp
self.fType = fileType
self.dType = None
self.fBeam = None
self.recordIndex = None
self.scanStartIndex = None
self.__filename = fileName
self.__filtered = filtered
self.__nocache = noCache
self.__src = src
self.__fd = None
self.__ptr = None
#check inputs
assert(isinstance(sTime,dt.datetime)), \
'error, sTime must be datetime object'
assert(eTime == None or isinstance(eTime,dt.datetime)), \
'error, eTime must be datetime object or None'
assert(channel == None or (isinstance(channel,str) and len(channel) == 1)), \
'error, channel must be None or a 1-letter string'
assert(bmnum == None or isinstance(bmnum,int)), \
'error, bmnum must be an int or None'
assert(cp == None or isinstance(cp,int)), \
'error, cp must be an int or None'
assert(fileType == 'rawacf' or fileType == 'fitacf' or \
fileType == 'fitex' or fileType == 'lmfit' or fileType == 'iqdat'), \
'error, fileType must be one of: rawacf,fitacf,fitex,lmfit,iqdat'
assert(fileName == None or isinstance(fileName,str)), \
'error, fileName must be None or a string'
assert(isinstance(filtered,bool)), \
'error, filtered must be True of False'
assert(src == None or src == 'local' or src == 'sftp'), \
'error, src must be one of None,local,sftp'
if radcode is not None:
assert(isinstance(radcode,str)), \
'error, radcode must be None or a string'
segments=radcode.split(".")
try: rad=segments[0]
except: rad=None
try: chan=segments[1]
except: chan=None
assert(isinstance(rad,str) and len(rad) == 3), \
'error, rad must be a 3 char string'
self.stid=int(network().getRadarByCode(rad).id)
if(self.eTime == None):
self.eTime = self.sTime+dt.timedelta(days=1)
filelist = []
if(fileType == 'fitex'): arr = ['fitex','fitacf','lmfit']
elif(fileType == 'fitacf'): arr = ['fitacf','fitex','lmfit']
elif(fileType == 'lmfit'): arr = ['lmfit','fitex','fitacf']
else: arr = [fileType]
#move back a little in time because files often start at 2 mins after the hour
sTime = sTime-dt.timedelta(minutes=4)
#a temporary directory to store a temporary file
try:
tmpDir=os.environ['DAVIT_TMPDIR']
except:
tmpDir = '/tmp/sd/'
d = os.path.dirname(tmpDir)
if not os.path.exists(d):
os.makedirs(d)
cached = False
fileSt = None
#FIRST, check if a specific filename was given
if fileName != None:
try:
if(not os.path.isfile(fileName)):
print 'problem reading',fileName,':file does not exist'
return None
outname = tmpDir+str(int(datetimeToEpoch(dt.datetime.now())))
if(string.find(fileName,'.bz2') != -1):
outname = string.replace(fileName,'.bz2','')
print 'bunzip2 -c '+fileName+' > '+outname+'\n'
os.system('bunzip2 -c '+fileName+' > '+outname)
elif(string.find(fileName,'.gz') != -1):
outname = string.replace(fileName,'.gz','')
print 'gunzip -c '+fileName+' > '+outname+'\n'
os.system('gunzip -c '+fileName+' > '+outname)
else:
os.system('cp '+fileName+' '+outname)
print 'cp '+fileName+' '+outname
filelist.append(outname)
self.dType = 'dmap'
fileSt = sTime
except Exception, e:
print e
print 'problem reading file',fileName
return None
def overlayFov(Basemap,
codes=None,
ids=None,
names=None,
dateTime=None,
all=False,
maxGate=None,
rangeLimits=None,
model='IS',
fovColor=None,
fovAlpha=0.2,
beams=None,
beamsColors=None,
hemi=None,
fovObj=None,
zorder=2,
lineColor='k',
lineWidth=1):
"""Overlay FoV position(s) on map
**Args**:
* **Basemap**: a python Basemap object on which to overplot the radar position(s)
* **[codes]**: a list of radar 3-letter codes to plot
* **[ids]**: a list of radar IDs to plot
* **[names]**: a list of radar names to plot
* **[dateTime]**: the date and time as a python datetime object
* **[all]**: set to true to plot all the radars (active ones)
* **[maxGate]**: Maximum number of gates to be plotted. Defaults to hdw.dat information.
* **[rangeLimits]**: (2-element list) Plot only between the range gates specified.
* **model**:
* **'IS'**: for ionopsheric scatter projection model (default)
* **'GS'**: for ground scatter projection model
* **None**: if you are really confident in your elevation or altitude values
* **[zorder]**: the overlay order number
* **[lineColor]**: FoV contour line color
* **[lineWidth]**: FoV contour line width
* **[fovColor]**: field of view fill color
* **[fovAlpha]**: field of view fill color transparency
* **[fovObj]**: a fov object. See pydarn.radar.radFov.fov
* **[hemi]**: 'north' or 'south', ignore radars from the other hemisphere
* **[beams]**: hightlight specified beams
* **[beamsColors]**: colors of the hightlighted beams
**Returns**:
* None
**Example**:
::
import pydarn, utils
width = 111e3*40
m = utils.plotUtils.mapObj(width=width, height=width, lat_0=50., lon_0=-95.)
# Plot radar position and code
pydarn.plot.overlayRadar(m, fontSize=12, codes='bks')
# Plot radar fov
pydarn.plot.overlayFov(m, codes='bks', maxGate=75, beams=[0,4,7,8,23])
written by Sebastien, 2012-09
"""
from pydarn.radar import network
from pydarn.radar.radFov import fov
from datetime import datetime as dt
from datetime import timedelta
import matplotlib.cm as cm
from numpy import transpose, ones, concatenate, vstack, shape
from matplotlib.patches import Polygon
from pylab import gca
# Set default date/time to now
if not dateTime:
dateTime = dt.utcnow()
# Load radar structure
NetworkObj = network()
# If all radars are to be plotted, create the list
if all: codes = NetworkObj.getAllCodes(datetime=dateTime, hemi=hemi)
# Define how the radars to be plotted are identified (code, id or name)
if codes:
input = {'meth': 'code', 'vals': codes}
elif ids:
input = {'meth': 'id', 'vals': ids}
elif names:
input = {'meth': 'name', 'vals': names}
else:
print 'overlayFov: no radars to plot'
return
# Check if radars is given as a list
if not isinstance(input['vals'], list): input['vals'] = [input['vals']]
nradars = len(input['vals'])
# iterates through radars to be plotted
for ir in xrange(nradars):
# Get field of view coordinates
if (fovObj == None):
rad = NetworkObj.getRadarBy(input['vals'][ir], input['meth'])
if not rad: continue
site = rad.getSiteByDate(dateTime)
if not site: continue
# Set number of gates to be plotted
eGate = site.maxgate - 1 if not maxGate else maxGate
if not hasattr(Basemap, 'coords'):
radFov = fov(site=site, ngates=eGate + 1, model=model)
else:
radFov = fov(site=site,
ngates=eGate + 1,
coords=Basemap.coords,
model=model)
else:
radFov = fovObj
eGate = len(fovObj.gates)
if rangeLimits is not None:
sGate = rangeLimits[0]
eGate = rangeLimits[1]
else:
sGate = 0
# Get radar coordinates in map projection
if hasattr(Basemap, 'coords'):
x, y = Basemap(radFov.lonFull,
radFov.latFull,
coords=radFov.coords)
else:
x, y = Basemap(radFov.lonFull, radFov.latFull)
# Plot field of view
# Create contour
contourX = concatenate(
(x[0, sGate:eGate], x[:, eGate], x[-1, eGate:sGate:-1], x[-1::-1,
sGate]))
contourY = concatenate(
(y[0, sGate:eGate], y[:, eGate], y[-1, eGate:sGate:-1], y[-1::-1,
sGate]))
# Plot contour
Basemap.plot(contourX,
contourY,
color=lineColor,
zorder=zorder,
linewidth=lineWidth)
# Field of view fill
if fovColor:
contour = transpose(vstack((contourX, contourY)))
patch = Polygon(contour,
color=fovColor,
alpha=fovAlpha,
zorder=zorder)
gca().add_patch(patch)
# Beams fill
if beams:
try:
[b for b in beams]
except:
beams = [beams]
for ib in beams:
if not (0 <= ib <= x.shape[0]): continue
if not beamsColors:
bColRGB = ib / float(x.shape[0])
bCol = (bColRGB / 2., bColRGB, 1)
else:
bCol = beamsColors[beams.index(ib)]
contourX = concatenate(
(x[ib, 0:eGate + 1], x[ib:ib + 2, eGate],
x[ib + 1, eGate::-1], x[ib + 1:ib - 1:-1, 0]))
contourY = concatenate(
(y[ib, 0:eGate + 1], y[ib:ib + 2, eGate],
y[ib + 1, eGate::-1], y[ib + 1:ib - 1:-1, 0]))
contour = transpose(vstack((contourX, contourY)))
patch = Polygon(contour, color=bCol, alpha=.4, zorder=zorder)
gca().add_patch(patch)
return
def overlayRadar(Basemap,
codes=None,
ids=None,
names=None,
dateTime=None,
annotate=True,
all=False,
hemi=None,
zorder=2,
markerColor='k',
markerSize=10,
fontSize=10,
xOffset=None):
"""Overlay radar position(s) and name(s) on map
**Args**:
* **Basemap**: a python Basemap object on which to overplot the radar position(s)
* **[codes]**: a list of radar 3-letter codes to plot
* **[ids]**: a list of radar IDs to plot
* **[names]**: a list of radar names to plot
* **[dateTime]**: the date and time as a python datetime object
* **[annotate]**: wether or not to show the radar(s) name(s)
* **[all]**: set to true to plot all the radars (active ones)
* **[hemi]**: 'north' or 'south', ignore radars from the other hemisphere
* **[zorder]**: the overlay order number
* **[markerColor]**:
* **[markerSize]**: [point]
* **[fontSize]**: [point]
* **[xOffset]**: x-Offset of the annotation in points
**Returns**:
* None
**Example**:
::
import pydarn, utils
m1 = utils.plotUtils.mapObj(boundinglat=30., gridLabels=True, coords='mag')
pydarn.plot.overlayRadar(m1, fontSize=8, all=True, markerSize=5)
written by Sebastien, 2012-08
"""
from pydarn.radar import network
from datetime import datetime as dt
from datetime import timedelta
from utils.plotUtils import textHighlighted
# Set default date/time to now
if not dateTime:
dateTime = dt.utcnow()
# Load radar structure
NetworkObj = network()
# If all radars are to be plotted, create the list
if all:
codes = NetworkObj.getAllCodes(datetime=dateTime, hemi=hemi)
# Define how the radars to be plotted are identified (code, id or name)
if codes:
input = {'meth': 'code', 'vals': codes}
elif ids:
input = {'meth': 'id', 'vals': ids}
elif names:
input = {'meth': 'name', 'vals': names}
else:
print 'overlayRadar: no radars to plot'
return
# Check if radars is given as a list
if not isinstance(input['vals'], list): input['vals'] = [input['vals']]
# Map width and height
width = Basemap.urcrnrx - Basemap.llcrnrx
height = Basemap.urcrnry - Basemap.llcrnry
if hemi is None:
hemiInt = 0
else:
hemiInt = 1 if hemi.lower()[0] == 'n' else -1
# iterates through radars to be plotted
for radN in input['vals']:
rad = NetworkObj.getRadarBy(radN, input['meth'])
if not rad: continue
site = rad.getSiteByDate(dateTime)
if not site: continue
# Check for hemisphere specification
if site.geolat * hemiInt < 0: continue
# Get radar coordinates in map projection
if not hasattr(Basemap, 'coords'):
x, y = Basemap(site.geolon, site.geolat)
else:
x, y = Basemap(site.geolon, site.geolat, coords='geo')
if not Basemap.xmin <= x <= Basemap.xmax: continue
if not Basemap.ymin <= y <= Basemap.ymax: continue
# Plot radar position
Basemap.scatter(x,
y,
s=markerSize,
marker='o',
color=markerColor,
zorder=zorder)
# Now add radar name
if annotate:
# If any of the other radar is too close...
if rad.code[0] in [
'adw', 'kod', 'cve', 'fhe', 'wal', 'gbr', 'pyk', 'aze',
'sys'
]:
xOff = 5 if not xOffset else xOffset
ha = 0
elif rad.code[0] in [
'ade', 'ksr', 'cvw', 'fhw', 'bks', 'sch', 'sto', 'azw',
'sye'
]:
xOff = -5 if not xOffset else xOffset
ha = 1
else:
xOff = 0.0
ha = .5
# Plot radar name
textHighlighted((x, y),
rad.code[0].upper(),
xytext=(xOff, -5),
text_alignment=(ha, 1),
variant='small-caps',
fontsize=fontSize,
zorder=zorder)
return
def overlayFov(Basemap, codes=None, ids=None, names=None,
dateTime=None, all=False,
maxGate=None, fovColor=None, fovAlpha=0.2,
beams=None, beamsColors=None, hemi=None, fovObj=None,
zorder=2, lineColor='k', lineWidth=1):
"""Overlay FoV position(s) on map
**Args**:
* **Basemap**: a python Basemap object on which to overplot the radar position(s)
* **[codes]**: a list of radar 3-letter codes to plot
* **[ids]**: a list of radar IDs to plot
* **[names]**: a list of radar names to plot
* **[dateTime]**: the date and time as a python datetime object
* **[all]**: set to true to plot all the radars (active ones)
* **[maxGate]**: Maximum number of gates to be plotted. Defaults to hdw.dat information.
* **[zorder]**: the overlay order number
* **[lineColor]**: FoV contour line color
* **[lineWidth]**: FoV contour line width
* **[fovColor]**: field of view fill color
* **[fovAlpha]**: field of view fill color transparency
* **[fovObj]**: a fov object. See pydarn.radar.radFov.fov
* **[hemi]**: 'north' or 'south', ignore radars from the other hemisphere
* **[beams]**: hightlight specified beams
* **[beamsColors]**: colors of the hightlighted beams
**Returns**:
* None
**Example**:
::
import pydarn, utils
width = 111e3*40
m = utils.plotUtils.mapObj(width=width, height=width, lat_0=50., lon_0=-95.)
# Plot radar position and code
pydarn.plot.overlayRadar(m, fontSize=12, codes='bks')
# Plot radar fov
pydarn.plot.overlayFov(m, codes='bks', maxGate=75, beams=[0,4,7,8,23])
written by Sebastien, 2012-09
"""
from pydarn.radar import network
from pydarn.radar.radFov import fov
from datetime import datetime as dt
from datetime import timedelta
import matplotlib.cm as cm
from numpy import transpose, ones, concatenate, vstack, shape
from matplotlib.patches import Polygon
from pylab import gca
# Set default date/time to now
if not dateTime:
dateTime = dt.utcnow()
# Load radar structure
NetworkObj = network()
# If all radars are to be plotted, create the list
if all: codes = NetworkObj.getAllCodes(datetime=dateTime, hemi=hemi)
# Define how the radars to be plotted are identified (code, id or name)
if codes:
input = {'meth': 'code', 'vals': codes}
elif ids:
input = {'meth': 'id', 'vals': ids}
elif names:
input = {'meth': 'name', 'vals': names}
else:
print 'overlayFov: no radars to plot'
return
# Check if radars is given as a list
if not isinstance(input['vals'], list): input['vals'] = [input['vals']]
nradars = len(input['vals'])
# iterates through radars to be plotted
for ir in xrange(nradars):
# Get field of view coordinates
if(fovObj == None):
rad = NetworkObj.getRadarBy(input['vals'][ir], input['meth'])
if not rad: continue
site = rad.getSiteByDate(dateTime)
if not site: continue
# Set number of gates to be plotted
eGate = site.maxgate-1 if not maxGate else maxGate
if not hasattr(Basemap, 'coords'):
radFov = fov(site=site, ngates=eGate+1)
else:
radFov = fov(site=site, ngates=eGate+1, coords=Basemap.coords)
else:
radFov = fovObj
eGate = len(fovObj.gates)
# Get radar coordinates in map projection
if hasattr(Basemap, 'coords'):
x, y = Basemap(radFov.lonFull, radFov.latFull, coords=radFov.coords)
else:
x, y = Basemap(radFov.lonFull, radFov.latFull)
# Plot field of view
# Create contour
contourX = concatenate( (x[0,0:eGate],
x[:,eGate],
x[-1,eGate::-1],
x[-1::-1,0]) )
contourY = concatenate( (y[0,0:eGate],
y[:,eGate],
y[-1,eGate::-1],
y[-1::-1,0]) )
# Plot contour
Basemap.plot(contourX, contourY,
color=lineColor, zorder=zorder, linewidth=lineWidth)
# Field of view fill
if fovColor:
contour = transpose( vstack((contourX,contourY)) )
patch = Polygon( contour, color=fovColor, alpha=fovAlpha, zorder=zorder)
gca().add_patch(patch)
# Beams fill
if beams:
try:
[b for b in beams]
except:
beams = [beams]
for ib in beams:
if not (0 <= ib <= x.shape[0]): continue
if not beamsColors:
bColRGB = ib/float(x.shape[0])
bCol = (bColRGB/2.,bColRGB,1)
else:
bCol = beamsColors[beams.index(ib)]
contourX = concatenate( (x[ib,0:eGate+1],
x[ib:ib+2,eGate],
x[ib+1,eGate::-1],
x[ib+1:ib-1:-1,0]) )
contourY = concatenate( (y[ib,0:eGate+1],
y[ib:ib+2,eGate],
y[ib+1,eGate::-1],
y[ib+1:ib-1:-1,0]) )
contour = transpose( vstack((contourX,contourY)) )
patch = Polygon( contour, color=bCol, alpha=.4, zorder=zorder)
gca().add_patch(patch)
return
def overlayRadar(Basemap, codes=None, ids=None, names=None, dateTime=None,
annotate=True, all=False, hemi=None,
zorder=2, markerColor='k', markerSize=10,
fontSize=10, xOffset=None):
"""Overlay radar position(s) and name(s) on map
**Args**:
* **Basemap**: a python Basemap object on which to overplot the radar position(s)
* **[codes]**: a list of radar 3-letter codes to plot
* **[ids]**: a list of radar IDs to plot
* **[names]**: a list of radar names to plot
* **[dateTime]**: the date and time as a python datetime object
* **[annotate]**: wether or not to show the radar(s) name(s)
* **[all]**: set to true to plot all the radars (active ones)
* **[hemi]**: 'north' or 'south', ignore radars from the other hemisphere
* **[zorder]**: the overlay order number
* **[markerColor]**:
* **[markerSize]**: [point]
* **[fontSize]**: [point]
* **[xOffset]**: x-Offset of the annotation in points
**Returns**:
* None
**Example**:
::
import pydarn, utils
m1 = utils.plotUtils.mapObj(boundinglat=30., gridLabels=True, coords='mag')
pydarn.plot.overlayRadar(m1, fontSize=8, all=True, markerSize=5)
written by Sebastien, 2012-08
"""
from pydarn.radar import network
from datetime import datetime as dt
from datetime import timedelta
from utils.plotUtils import textHighlighted
# Set default date/time to now
if not dateTime:
dateTime = dt.utcnow()
# Load radar structure
NetworkObj = network()
# If all radars are to be plotted, create the list
if all:
codes = NetworkObj.getAllCodes(datetime=dateTime, hemi=hemi)
# Define how the radars to be plotted are identified (code, id or name)
if codes:
input = {'meth': 'code', 'vals': codes}
elif ids:
input = {'meth': 'id', 'vals': ids}
elif names:
input = {'meth': 'name', 'vals': names}
else:
print 'overlayRadar: no radars to plot'
return
# Check if radars is given as a list
if not isinstance(input['vals'], list): input['vals'] = [input['vals']]
# Map width and height
width = Basemap.urcrnrx - Basemap.llcrnrx
height = Basemap.urcrnry - Basemap.llcrnry
if hemi is None:
hemiInt = 0
else:
hemiInt = 1 if hemi.lower()[0]=='n' else -1
# iterates through radars to be plotted
for radN in input['vals']:
rad = NetworkObj.getRadarBy(radN, input['meth'])
if not rad: continue
site = rad.getSiteByDate(dateTime)
if not site: continue
# Check for hemisphere specification
if site.geolat*hemiInt < 0: continue
# Get radar coordinates in map projection
if not hasattr(Basemap, 'coords'):
x,y = Basemap(site.geolon, site.geolat)
else:
x,y = Basemap(site.geolon, site.geolat, coords='geo')
if not Basemap.xmin <= x <= Basemap.xmax: continue
if not Basemap.ymin <= y <= Basemap.ymax: continue
# Plot radar position
Basemap.scatter(x, y, s=markerSize, marker='o', color=markerColor, zorder=zorder)
# Now add radar name
if annotate:
# If any of the other radar is too close...
if rad.code[0] in ['adw', 'kod', 'cve', 'fhe', 'wal', 'gbr', 'pyk', 'aze', 'sys']:
xOff = 5 if not xOffset else xOffset
ha = 0
elif rad.code[0] in ['ade', 'ksr', 'cvw', 'fhw', 'bks', 'sch', 'sto', 'azw', 'sye']:
xOff = -5 if not xOffset else xOffset
ha = 1
else:
xOff = 0.0
ha = .5
# Plot radar name
textHighlighted((x, y), rad.code[0].upper(), xytext=(xOff, -5),
text_alignment=(ha,1), variant='small-caps', fontsize=fontSize, zorder=zorder)
return
def radDataOpen(sTime,radcode,eTime=None,channel=None,bmnum=None,cp=None, \
fileType='fitex',filtered=False, src=None,fileName=None, \
custType='fitex',noCache=False):
"""A function to establish a pipeline through which we can read radar data. first it tries the mongodb, then it tries to find local files, and lastly it sftp's over to the VT data server.
**Args**:
* **sTime** (`datetime <http://tinyurl.com/bl352yx>`_): the beginning time for which you want data
* **radcode** (str): the 3-letter radar code with optional channel extension for which you want data
* **[eTime]** (`datetime <http://tinyurl.com/bl352yx>`_): the last time that you want data for. if this is set to None, it will be set to 1 day after sTime. default = None
* **[channel]** (str): the 1-letter code for what channel you want data from, eg 'a','b',... if this is set to None, data from ALL channels will be read. default = None
* **[bmnum]** (int): the beam number which you want data for. If this is set to None, data from all beams will be read. default = None
* **[cp]** (int): the control program which you want data for. If this is set to None, data from all cp's will be read. default = None
* **[fileType]** (str): The type of data you want to read. valid inputs are: 'fitex','fitacf','rawacf','iqdat'. if you choose a fit file format and the specified one isn't found, we will search for one of the others. Beware: if you ask for rawacf/iq data, these files are large and the data transfer might take a long time. default = 'fitex'
* **[filtered]** (boolean): a boolean specifying whether you want the fit data to be boxcar filtered. ONLY VALID FOR FIT. default = False
* **[src]** (str): the source of the data. valid inputs are 'local' 'sftp'. if this is set to None, it will try all possibilites sequentially. default = None
* **[fileName]** (str): the name of a specific file which you want to open. default=None
* **[custType]** (str): if fileName is specified, the filetype of the file. default='fitex'
* **[noCache]** (boolean): flag to indicate that you do not want to check first for cached files. default = False.
**Returns**:
* **myPtr** (:class:`pydarn.sdio.radDataTypes.radDataPtr`): a radDataPtr object which contains a link to the data to be read. this can then be passed to radDataReadRec in order to actually read the data.
**ENVIRONMENT Variables**:
* DAVIT_TMPDIR : Directory used for davitpy temporary file cache.
* DAVIT_TMPEXPIRE : Length of time that cached temporary files are valid. After which they will be regenerated. Example: DAVIT_TMPEXPIRE='2h' will reuse temp files in the cache for 2 hours since last access
* DAVIT_LOCALDIR : Used to set base directory tree for local file look up
* DAVIT_DIRFORMAT : Python string dictionary capable format string appended to local file base directory tree for use with directory structures which encode radar name, channel or date information.
Currently supported dictionary keys which can be used:
"dirtree" : base directory tree
"year" : 0 padded 4 digit year
"month" : 0 padded 2 digit month
"day" : 0 padded 2 digit day
"ftype" : filetype string
"radar" : 3-chr radarcode
**Example**:
::
import datetime as dt
myPtr = radDataOpen(dt.datetime(2011,1,1),'bks',eTime=dt.datetime(2011,1,1,2),channel='a', bmnum=7,cp=153,fileType='fitex',filtered=False, src=None):
Written by AJ 20130110
"""
import paramiko as p
import re
import string
import datetime as dt, os, pydarn.sdio, glob
from pydarn.sdio import radDataPtr
from pydarn.radar import network
from utils.timeUtils import datetimeToEpoch
#check inputs
assert(isinstance(sTime,dt.datetime)), \
'error, sTime must be datetime object'
segments=radcode.split(".")
try: rad=segments[0]
except: rad=None
try: chan=segments[1]
except: chan=None
assert(isinstance(rad,str) and len(rad) == 3), \
'error, rad must be a 3 char string'
assert(eTime == None or isinstance(eTime,dt.datetime)), \
'error, eTime must be datetime object or None'
assert(channel == None or (isinstance(channel,str) and len(channel) == 1)), \
'error, channel must be None or a 1-letter string'
assert(bmnum == None or isinstance(bmnum,int)), \
'error, bmnum must be an int or None'
assert(cp == None or isinstance(cp,int)), \
'error, cp must be an int or None'
assert(fileType == 'rawacf' or fileType == 'fitacf' or \
fileType == 'fitex' or fileType == 'iqdat'), \
'error, fileType must be one of: rawacf,fitacf,fitex,iqdat'
assert(fileName == None or isinstance(fileName,str)), \
'error, fileName must be None or a string'
assert(isinstance(filtered,bool)), \
'error, filtered must be True of False'
assert(src == None or src == 'local' or src == 'sftp'), \
'error, src must be one of None,local,sftp'
if(eTime == None):
eTime = sTime+dt.timedelta(days=1)
#create a datapointer object
myPtr = radDataPtr(sTime=sTime,eTime=eTime,stid=int(network().getRadarByCode(rad).id),
channel=channel,bmnum=bmnum,cp=cp)
filelist = []
# if(fileType == 'fitex'): arr = ['fitex','fitacf','lmfit']
# elif(fileType == 'fitacf'): arr = ['fitacf','fitex','lmfit']
# elif(fileType == 'lmfit'): arr = ['lmfit','fitex','fitacf']
if(fileType == 'fitex'): arr = ['fitex','fitacf']
elif(fileType == 'fitacf'): arr = ['fitacf','fitex']
else: arr = [fileType]
#move back a little in time because files often start at 2 mins after the hour
sTime = sTime-dt.timedelta(minutes=4)
#a temporary directory to store a temporary file
try:
tmpDir=os.environ['DAVIT_TMPDIR']
except:
tmpDir = '/tmp/sd/'
d = os.path.dirname(tmpDir)
if not os.path.exists(d):
os.makedirs(d)
cached = False
fileSt = None
#FIRST, check if a specific filename was given
if fileName != None:
try:
if(not os.path.isfile(fileName)):
print 'problem reading',fileName,':file does not exist'
return None
outname = tmpDir+str(int(datetimeToEpoch(dt.datetime.now())))
if(string.find(fileName,'.bz2') != -1):
outname = string.replace(fileName,'.bz2','')
print 'bunzip2 -c '+fileName+' > '+outname+'\n'
os.system('bunzip2 -c '+fileName+' > '+outname)
elif(string.find(fileName,'.gz') != -1):
outname = string.replace(fileName,'.gz','')
print 'gunzip -c '+fileName+' > '+outname+'\n'
os.system('gunzip -c '+fileName+' > '+outname)
else:
os.system('cp '+fileName+' '+outname)
print 'cp '+fileName+' '+outname
filelist.append(outname)
myPtr.fType,myPtr.dType = custType,'dmap'
fileSt = sTime
except Exception, e:
print e
print 'problem reading file',fileName
return None
def radDataOpen(sTime,radcode,eTime=None,channel=None,bmnum=None,cp=None, \
fileType='fitex',filtered=False, src=None,fileName=None, \
custType='fitex',noCache=False):
"""A function to establish a pipeline through which we can read radar data. first it tries the mongodb, then it tries to find local files, and lastly it sftp's over to the VT data server.
**Args**:
* **sTime** (`datetime <http://tinyurl.com/bl352yx>`_): the beginning time for which you want data
* **radcode** (str): the 3-letter radar code with optional channel extension for which you want data
* **[eTime]** (`datetime <http://tinyurl.com/bl352yx>`_): the last time that you want data for. if this is set to None, it will be set to 1 day after sTime. default = None
* **[channel]** (str): the 1-letter code for what channel you want data from, eg 'a','b',... if this is set to None, data from ALL channels will be read. default = None
* **[bmnum]** (int): the beam number which you want data for. If this is set to None, data from all beams will be read. default = None
* **[cp]** (int): the control program which you want data for. If this is set to None, data from all cp's will be read. default = None
* **[fileType]** (str): The type of data you want to read. valid inputs are: 'fitex','fitacf','rawacf','iqdat'. if you choose a fit file format and the specified one isn't found, we will search for one of the others. Beware: if you ask for rawacf/iq data, these files are large and the data transfer might take a long time. default = 'fitex'
* **[filtered]** (boolean): a boolean specifying whether you want the fit data to be boxcar filtered. ONLY VALID FOR FIT. default = False
* **[src]** (str): the source of the data. valid inputs are 'local' 'sftp'. if this is set to None, it will try all possibilites sequentially. default = None
* **[fileName]** (str): the name of a specific file which you want to open. default=None
* **[custType]** (str): if fileName is specified, the filetype of the file. default='fitex'
* **[noCache]** (boolean): flag to indicate that you do not want to check first for cached files. default = False.
**Returns**:
* **myPtr** (:class:`pydarn.sdio.radDataTypes.radDataPtr`): a radDataPtr object which contains a link to the data to be read. this can then be passed to radDataReadRec in order to actually read the data.
**ENVIRONMENT Variables**:
* DAVIT_TMPDIR : Directory used for davitpy temporary file cache.
* DAVIT_TMPEXPIRE : Length of time that cached temporary files are valid. After which they will be regenerated. Example: DAVIT_TMPEXPIRE='2h' will reuse temp files in the cache for 2 hours since last access
* DAVIT_LOCALDIR : Used to set base directory tree for local file look up
* DAVIT_DIRFORMAT : Python string dictionary capable format string appended to local file base directory tree for use with directory structures which encode radar name, channel or date information.
Currently supported dictionary keys which can be used:
"dirtree" : base directory tree
"year" : 0 padded 4 digit year
"month" : 0 padded 2 digit month
"day" : 0 padded 2 digit day
"ftype" : filetype string
"radar" : 3-chr radarcode
**Example**:
::
import datetime as dt
myPtr = radDataOpen(dt.datetime(2011,1,1),'bks',eTime=dt.datetime(2011,1,1,2),channel='a', bmnum=7,cp=153,fileType='fitex',filtered=False, src=None):
Written by AJ 20130110
"""
import paramiko as p
import re
import string
import datetime as dt, os, pydarn.sdio, glob
from pydarn.sdio import radDataPtr
from pydarn.radar import network
from utils.timeUtils import datetimeToEpoch
#check inputs
assert(isinstance(sTime,dt.datetime)), \
'error, sTime must be datetime object'
segments = radcode.split(".")
try:
rad = segments[0]
except:
rad = None
try:
chan = segments[1]
except:
chan = None
assert(isinstance(rad,str) and len(rad) == 3), \
'error, rad must be a 3 char string'
assert(eTime == None or isinstance(eTime,dt.datetime)), \
'error, eTime must be datetime object or None'
assert(channel == None or (isinstance(channel,str) and len(channel) == 1)), \
'error, channel must be None or a 1-letter string'
assert(bmnum == None or isinstance(bmnum,int)), \
'error, bmnum must be an int or None'
assert(cp == None or isinstance(cp,int)), \
'error, cp must be an int or None'
assert(fileType == 'rawacf' or fileType == 'fitacf' or \
fileType == 'fitex' or fileType == 'iqdat'), \
'error, fileType must be one of: rawacf,fitacf,fitex,iqdat'
assert(fileName == None or isinstance(fileName,str)), \
'error, fileName must be None or a string'
assert(isinstance(filtered,bool)), \
'error, filtered must be True of False'
assert(src == None or src == 'local' or src == 'sftp'), \
'error, src must be one of None,local,sftp'
if (eTime == None):
eTime = sTime + dt.timedelta(days=1)
#create a datapointer object
myPtr = radDataPtr(sTime=sTime,
eTime=eTime,
stid=int(network().getRadarByCode(rad).id),
channel=channel,
bmnum=bmnum,
cp=cp)
filelist = []
# if(fileType == 'fitex'): arr = ['fitex','fitacf','lmfit']
# elif(fileType == 'fitacf'): arr = ['fitacf','fitex','lmfit']
# elif(fileType == 'lmfit'): arr = ['lmfit','fitex','fitacf']
if (fileType == 'fitex'): arr = ['fitex', 'fitacf']
elif (fileType == 'fitacf'): arr = ['fitacf', 'fitex']
else: arr = [fileType]
#move back a little in time because files often start at 2 mins after the hour
sTime = sTime - dt.timedelta(minutes=4)
#a temporary directory to store a temporary file
try:
tmpDir = os.environ['DAVIT_TMPDIR']
except:
tmpDir = '/tmp/sd/'
d = os.path.dirname(tmpDir)
if not os.path.exists(d):
os.makedirs(d)
cached = False
fileSt = None
#FIRST, check if a specific filename was given
if fileName != None:
try:
if (not os.path.isfile(fileName)):
print 'problem reading', fileName, ':file does not exist'
return None
outname = tmpDir + str(int(datetimeToEpoch(dt.datetime.now())))
if (string.find(fileName, '.bz2') != -1):
outname = string.replace(fileName, '.bz2', '')
print 'bunzip2 -c ' + fileName + ' > ' + outname + '\n'
os.system('bunzip2 -c ' + fileName + ' > ' + outname)
elif (string.find(fileName, '.gz') != -1):
outname = string.replace(fileName, '.gz', '')
print 'gunzip -c ' + fileName + ' > ' + outname + '\n'
os.system('gunzip -c ' + fileName + ' > ' + outname)
else:
os.system('cp ' + fileName + ' ' + outname)
print 'cp ' + fileName + ' ' + outname
filelist.append(outname)
myPtr.fType, myPtr.dType = custType, 'dmap'
fileSt = sTime
except Exception, e:
print e
print 'problem reading file', fileName
return None
def plot(date=None, traceA=None, traceB=None, saveDb=True, noPlot=False):
"""
Plot RBSP footprints
"""
assert not date == None or not (
traceA == None
and traceB == None), 'Either give a date, or a trace for A and B'
datapath = '/home/sebastien/Documents/code/rbsp/data/'
imgpath = '/home/sebastien/Documents/code/rbsp/img/'
# Read trace
if not traceA:
traceA = ts.tsygTrace(filename=datapath +
'trace.{}.A.dat'.format(date.strftime('%Y%m%d')))
if not traceB:
traceB = ts.tsygTrace(filename=datapath +
'trace.{}.B.dat'.format(date.strftime('%Y%m%d')))
if not date:
date = traceA.datetime[0]
if not noPlot:
# Init figure
fig = pylab.figure(figsize=(11, 6.5))
pylab.rcParams.update({'font.size': 14})
fovCol = (.5, .6, .6)
fovAlpha = .2
# NH
ax1 = fig.add_subplot(121)
m1 = Basemap(projection='npstere',
boundinglat=35,
lon_0=270,
resolution='l')
m1.drawcoastlines(color='.5')
m1.fillcontinents(color=(.8, .8, .8))
# draw parallels and meridians.
m1.drawparallels(np.arange(-80., 81., 20.), color='.6')
m1.drawmeridians(np.arange(-180., 181., 20.), color='.6')
m1.drawmapboundary()
# Draw FP
x, y = m1(traceA.lonNH, traceA.latNH)
m1.scatter(x, y, color='r', zorder=3, s=5)
x, y = m1(traceB.lonNH, traceB.latNH)
m1.scatter(x, y, color='b', zorder=3, s=5)
overlayFov(m1,
all=True,
hemi='north',
maxGate=75,
dateTime=traceA.datetime[0],
fovColor=fovCol,
lineColor=fovCol,
lineWidth=0,
fovAlpha=fovAlpha)
# SH
ax2 = fig.add_subplot(122)
m2 = Basemap(projection='spstere',
boundinglat=-35,
lon_0=270,
resolution='l')
m2.drawcoastlines(color='.5')
m2.fillcontinents(color=(.8, .8, .8))
# draw parallels and meridians.
m2.drawparallels(np.arange(-80., 81., 20.), color='.6')
m2.drawmeridians(np.arange(-180., 181., 20.), color='.6')
m2.drawmapboundary()
# Draw FP
x, y = m2(traceA.lonSH, traceA.latSH)
m2.scatter(x, y, color='r', zorder=3, s=5)
x, y = m2(traceB.lonSH, traceB.latSH)
m2.scatter(x, y, color='b', zorder=3, s=5)
overlayFov(m2,
all=True,
hemi='south',
maxGate=75,
dateTime=traceA.datetime[0],
fovColor=fovCol,
lineColor=fovCol,
lineWidth=0,
fovAlpha=fovAlpha)
fig.tight_layout()
Title = '{}'.format(date.strftime('%Y - %b - %d'))
fig.text(.02, .92, Title)
fig.text(.02, .09, 'VAP-A', color='r', va='top')
fig.text(.1, .09, 'VAP-B', color='b', va='top')
Title = '{}'.format(date.strftime('%Y - %b - %d'))
fig.text(.51, .92, Title)
fig.text(.51, .09, 'VAP-A', color='r', va='top')
fig.text(.59, .09, 'VAP-B', color='b', va='top')
# List apogees
minsA = np.r_[True, traceA.rho[1:] >= traceA.rho[:-1]] & np.r_[
traceA.rho[:-1] > traceA.rho[1:], True]
minsA[0] = minsA[-1] = False
lonANH = traceA.lonNH[minsA]
latANH = traceA.latNH[minsA]
lonASH = traceA.lonSH[minsA]
latASH = traceA.latSH[minsA]
timeA = traceA.datetime[minsA]
minsB = np.r_[True, traceB.rho[1:] >= traceB.rho[:-1]] & np.r_[
traceB.rho[:-1] > traceB.rho[1:], True]
minsB[0] = minsB[-1] = False
lonBNH = traceB.lonNH[minsB]
latBNH = traceB.latNH[minsB]
lonBSH = traceB.lonSH[minsB]
latBSH = traceB.latSH[minsB]
timeB = traceB.datetime[minsB]
rads = network()
apoA_NH = ''
apoA_SH = ''
radsANH = []
radsASH = []
for i in xrange(len(lonANH)):
apoA_NH += '({}UT, {:4.2f}E, {:4.2f}N) | '.format(
timeA[i].strftime('%H:%M'), lonANH[i], latANH[i])
tradANH = rads.getRadarsByPosition(latANH[i],
lonANH[i],
300.,
datetime=timeA[i])
if not noPlot:
x, y = m1(lonANH[i], latANH[i])
# ax1.text(x, y, timeA[i].strftime('%H:%M'), zorder=5, clip_on=True, color=(.3,0,0), fontsize=10, ha='center')
textHighlighted((x, y),
timeA[i].strftime('%H:%M'),
ax=ax1,
zorder=6,
color=(.3, 0, 0),
fontsize=10)
if tradANH:
radsANH.append(tradANH)
if saveDb: writeToDb(timeA[i], latANH[i], lonANH[i], 'A', tradANH)
apoA_SH += '({}UT, {:4.2f}E, {:4.2f}N) | '.format(
timeA[i].strftime('%H:%M'), lonASH[i], latASH[i])
tradASH = rads.getRadarsByPosition(latASH[i],
lonASH[i],
300.,
datetime=timeA[i])
if not noPlot:
x, y = m2(lonASH[i], latASH[i])
# ax2.text(x, y, timeA[i].strftime('%H:%M'), zorder=5, clip_on=True, color=(.3,0,0), fontsize=10, ha='center')
textHighlighted((x, y),
timeA[i].strftime('%H:%M'),
ax=ax2,
zorder=6,
color=(.3, 0, 0),
fontsize=10)
if tradASH:
radsASH.append(tradASH)
if saveDb: writeToDb(timeA[i], latASH[i], lonASH[i], 'A', tradASH)
apoB_NH = ''
apoB_SH = ''
radsBNH = []
radsBSH = []
for i in xrange(len(lonBNH)):
apoB_NH += '({}UT, {:4.2f}E, {:4.2f}N) '.format(
timeB[i].strftime('%H:%M'), lonBNH[i], latBNH[i])
tradBNH = rads.getRadarsByPosition(latBNH[i],
lonBNH[i],
300.,
datetime=timeB[i])
if not noPlot:
x, y = m1(lonBNH[i], latBNH[i])
# ax1.text(x, y, timeB[i].strftime('%H:%M'), zorder=5, clip_on=True, color=(0,0,.3), fontsize=10, ha='center')
textHighlighted((x, y),
timeB[i].strftime('%H:%M'),
ax=ax1,
zorder=6,
color=(0, 0, .3),
fontsize=10)
if tradBNH:
radsBNH.append(tradBNH)
if saveDb: writeToDb(timeB[i], latBNH[i], lonBNH[i], 'B', tradBNH)
apoB_SH += '({}UT, {:4.2f}E, {:4.2f}N) '.format(
timeB[i].strftime('%H:%M'), lonBSH[i], latBSH[i])
tradBSH = rads.getRadarsByPosition(latBSH[i],
lonBSH[i],
300.,
datetime=timeB[i])
if not noPlot:
x, y = m2(lonBSH[i], latBSH[i])
# ax2.text(x, y, timeB[i].strftime('%H:%M'), zorder=5, clip_on=True, color=(0,0,.3), fontsize=10, ha='center')
textHighlighted((x, y),
timeB[i].strftime('%H:%M'),
ax=ax2,
zorder=6,
color=(0, 0, .3),
fontsize=10)
if tradBSH:
radsBSH.append(tradBSH)
if saveDb: writeToDb(timeB[i], latBSH[i], lonBSH[i], 'B', tradBSH)
if not noPlot:
fig.savefig(imgpath +
'rbsp.map.{}.pdf'.format(date.strftime('%Y%m%d')))
fig.savefig(imgpath +
'rbsp.map.{}.svg'.format(date.strftime('%Y%m%d')))
if __name__ == '__main__':
fig.clf()
pylab.close(fig)
def plot(date=None, traceA=None, traceB=None, saveDb=True, noPlot=False):
"""
Plot RBSP footprints
"""
assert not date==None or not (traceA==None and traceB==None), 'Either give a date, or a trace for A and B'
datapath = '/home/sebastien/Documents/code/rbsp/data/'
imgpath = '/home/sebastien/Documents/code/rbsp/img/'
# Read trace
if not traceA:
traceA = ts.tsygTrace(filename=datapath+'trace.{}.A.dat'.format(date.strftime('%Y%m%d')))
if not traceB:
traceB = ts.tsygTrace(filename=datapath+'trace.{}.B.dat'.format(date.strftime('%Y%m%d')))
if not date:
date = traceA.datetime[0]
if not noPlot:
# Init figure
fig = pylab.figure(figsize=(11,6.5))
pylab.rcParams.update({'font.size': 14})
fovCol = (.5,.6,.6)
fovAlpha = .2
# NH
ax1 = fig.add_subplot(121)
m1 = Basemap(projection='npstere',boundinglat=35,lon_0=270,resolution='l')
m1.drawcoastlines(color='.5')
m1.fillcontinents(color=(.8,.8,.8))
# draw parallels and meridians.
m1.drawparallels(np.arange(-80.,81.,20.), color='.6')
m1.drawmeridians(np.arange(-180.,181.,20.), color='.6')
m1.drawmapboundary()
# Draw FP
x, y = m1(traceA.lonNH, traceA.latNH)
m1.scatter(x, y, color='r', zorder=3, s=5)
x, y = m1(traceB.lonNH, traceB.latNH)
m1.scatter(x, y, color='b', zorder=3, s=5)
overlayFov(m1, all=True, hemi='north', maxGate=75, dateTime=traceA.datetime[0],
fovColor=fovCol, lineColor=fovCol, lineWidth=0, fovAlpha=fovAlpha)
# SH
ax2 = fig.add_subplot(122)
m2 = Basemap(projection='spstere',boundinglat=-35,lon_0=270,resolution='l')
m2.drawcoastlines(color='.5')
m2.fillcontinents(color=(.8,.8,.8))
# draw parallels and meridians.
m2.drawparallels(np.arange(-80.,81.,20.), color='.6')
m2.drawmeridians(np.arange(-180.,181.,20.), color='.6')
m2.drawmapboundary()
# Draw FP
x, y = m2(traceA.lonSH, traceA.latSH)
m2.scatter(x, y, color='r', zorder=3, s=5)
x, y = m2(traceB.lonSH, traceB.latSH)
m2.scatter(x, y, color='b', zorder=3, s=5)
overlayFov(m2, all=True, hemi='south', maxGate=75, dateTime=traceA.datetime[0],
fovColor=fovCol, lineColor=fovCol, lineWidth=0, fovAlpha=fovAlpha)
fig.tight_layout()
Title = '{}'.format(date.strftime('%Y - %b - %d'))
fig.text(.02, .92, Title)
fig.text(.02, .09, 'VAP-A', color='r', va='top')
fig.text(.1, .09, 'VAP-B', color='b', va='top')
Title = '{}'.format(date.strftime('%Y - %b - %d'))
fig.text(.51, .92, Title)
fig.text(.51, .09, 'VAP-A', color='r', va='top')
fig.text(.59, .09, 'VAP-B', color='b', va='top')
# List apogees
minsA = np.r_[True, traceA.rho[1:] >= traceA.rho[:-1]] & np.r_[traceA.rho[:-1] > traceA.rho[1:], True]
minsA[0] = minsA[-1] = False
lonANH = traceA.lonNH[minsA]
latANH = traceA.latNH[minsA]
lonASH = traceA.lonSH[minsA]
latASH = traceA.latSH[minsA]
timeA = traceA.datetime[minsA]
minsB = np.r_[True, traceB.rho[1:] >= traceB.rho[:-1]] & np.r_[traceB.rho[:-1] > traceB.rho[1:], True]
minsB[0] = minsB[-1] = False
lonBNH = traceB.lonNH[minsB]
latBNH = traceB.latNH[minsB]
lonBSH = traceB.lonSH[minsB]
latBSH = traceB.latSH[minsB]
timeB = traceB.datetime[minsB]
rads = network()
apoA_NH = ''
apoA_SH = ''
radsANH = []
radsASH = []
for i in xrange(len(lonANH)):
apoA_NH += '({}UT, {:4.2f}E, {:4.2f}N) | '.format(timeA[i].strftime('%H:%M'), lonANH[i], latANH[i])
tradANH = rads.getRadarsByPosition(latANH[i], lonANH[i], 300., datetime=timeA[i])
if not noPlot:
x, y = m1(lonANH[i], latANH[i])
# ax1.text(x, y, timeA[i].strftime('%H:%M'), zorder=5, clip_on=True, color=(.3,0,0), fontsize=10, ha='center')
textHighlighted((x,y), timeA[i].strftime('%H:%M'), ax=ax1, zorder=6, color=(.3,0,0), fontsize=10)
if tradANH:
radsANH.append( tradANH )
if saveDb: writeToDb(timeA[i], latANH[i], lonANH[i], 'A', tradANH)
apoA_SH += '({}UT, {:4.2f}E, {:4.2f}N) | '.format(timeA[i].strftime('%H:%M'), lonASH[i], latASH[i])
tradASH = rads.getRadarsByPosition(latASH[i], lonASH[i], 300., datetime=timeA[i])
if not noPlot:
x, y = m2(lonASH[i], latASH[i])
# ax2.text(x, y, timeA[i].strftime('%H:%M'), zorder=5, clip_on=True, color=(.3,0,0), fontsize=10, ha='center')
textHighlighted((x,y), timeA[i].strftime('%H:%M'), ax=ax2, zorder=6, color=(.3,0,0), fontsize=10)
if tradASH:
radsASH.append( tradASH )
if saveDb: writeToDb(timeA[i], latASH[i], lonASH[i], 'A', tradASH)
apoB_NH = ''
apoB_SH = ''
radsBNH = []
radsBSH = []
for i in xrange(len(lonBNH)):
apoB_NH += '({}UT, {:4.2f}E, {:4.2f}N) '.format(timeB[i].strftime('%H:%M'), lonBNH[i], latBNH[i])
tradBNH = rads.getRadarsByPosition(latBNH[i], lonBNH[i], 300., datetime=timeB[i])
if not noPlot:
x, y = m1(lonBNH[i], latBNH[i])
# ax1.text(x, y, timeB[i].strftime('%H:%M'), zorder=5, clip_on=True, color=(0,0,.3), fontsize=10, ha='center')
textHighlighted((x,y), timeB[i].strftime('%H:%M'), ax=ax1, zorder=6, color=(0,0,.3), fontsize=10)
if tradBNH:
radsBNH.append( tradBNH )
if saveDb: writeToDb(timeB[i], latBNH[i], lonBNH[i], 'B', tradBNH)
apoB_SH += '({}UT, {:4.2f}E, {:4.2f}N) '.format(timeB[i].strftime('%H:%M'), lonBSH[i], latBSH[i])
tradBSH = rads.getRadarsByPosition(latBSH[i], lonBSH[i], 300., datetime=timeB[i])
if not noPlot:
x, y = m2(lonBSH[i], latBSH[i])
# ax2.text(x, y, timeB[i].strftime('%H:%M'), zorder=5, clip_on=True, color=(0,0,.3), fontsize=10, ha='center')
textHighlighted((x,y), timeB[i].strftime('%H:%M'), ax=ax2, zorder=6, color=(0,0,.3), fontsize=10)
if tradBSH:
radsBSH.append( tradBSH )
if saveDb: writeToDb(timeB[i], latBSH[i], lonBSH[i], 'B', tradBSH)
if not noPlot:
fig.savefig( imgpath+'rbsp.map.{}.pdf'.format(date.strftime('%Y%m%d')) )
fig.savefig( imgpath+'rbsp.map.{}.svg'.format(date.strftime('%Y%m%d')) )
if __name__ == '__main__':
fig.clf()
pylab.close(fig)
def radDataOpen(sTime,rad,eTime=None,channel=None,bmnum=None,cp=None, \
fileType='fitex',filtered=False, src=None,fileName=None, \
custType='fitex'):
"""A function to establish a pipeline through which we can read radar data. first it tries the mongodb, then it tries to find local files, and lastly it sftp's over to the VT data server.
**Args**:
* **sTime** (`datetime <http://tinyurl.com/bl352yx>`_): the beginning time for which you want data
* **rad** (str): the 3-letter radar code for which you want data
* **[eTime]** (`datetime <http://tinyurl.com/bl352yx>`_): the last time that you want data for. if this is set to None, it will be set to 1 day after sTime. default = None
* **[channel]** (str): the 1-letter code for what channel you want data from, eg 'a','b',... if this is set to None, data from ALL channels will be read. default = None
* **[bmnum]** (int): the beam number which you want data for. If this is set to None, data from all beams will be read. default = None
* **[cp]** (int): the control program which you want data for. If this is set to None, data from all cp's will be read. default = None
* **[fileType]** (str): The type of data you want to read. valid inputs are: 'fitex','fitacf','lmfit','rawacf','iqdat'. if you choose a fit file format and the specified one isn't found, we will search for one of the others. Beware: if you ask for rawacf/iq data, these files are large and the data transfer might take a long time. default = 'fitex'
* **[filtered]** (boolean): a boolean specifying whether you want the fit data to be boxcar filtered. ONLY VALID FOR FIT. default = False
* **[src]** (str): the source of the data. valid inputs are 'mongo' 'local' 'sftp'. if this is set to None, it will try all possibilites sequentially. default = None
* **[fileName]** (str): the name of a specific file which you want to open. default=None
* **[custType]** (str): if fileName is specified, the filetype of the file. default='fitex'
**Returns**:
* **myPtr** (:class:`radDataTypes.radDataPtr`): a radDataPtr object which contains a link to the data to be read. this can then be passed to radDataReadRec in order to actually read the data.
**Example**:
::
import datetime as dt
myPtr = radDataOpen(dt.datetime(2011,1,1),'bks',eTime=dt.datetime(2011,1,1,2),channel='a', bmnum=7,cp=153,fileType='fitex',filtered=False, src=None):
Written by AJ 20130110
"""
import subprocess as sub, paramiko as p, re, string
import datetime as dt, os, pydarn.sdio, glob
from pydarn.sdio import radDataPtr
from pydarn.radar import network
from utils.timeUtils import datetimeToEpoch
#check inputs
assert(isinstance(sTime,dt.datetime)), \
'error, sTime must be datetime object'
assert(isinstance(rad,str) and len(rad) == 3), \
'error, rad must be a 3 char string'
assert(eTime == None or isinstance(eTime,dt.datetime)), \
'error, eTime must be datetime object or None'
assert(channel == None or (isinstance(channel,str) and len(channel) == 1)), \
'error, channel must be None or a 1-letter string'
assert(bmnum == None or isinstance(bmnum,int)), \
'error, bmnum must be an int or None'
assert(cp == None or isinstance(cp,int)), \
'error, cp must be an int or None'
assert(fileType == 'rawacf' or fileType == 'fitacf' or \
fileType == 'fitex' or fileType == 'lmfit' or fileType == 'iqdat'), \
'error, fileType must be one of: rawacf,fitacf,fitex,lmfit,iqdat'
assert(fileName == None or isinstance(fileName,str)), \
'error, fileName must be None or a string'
assert(isinstance(filtered,bool)), \
'error, filtered must be True of False'
assert(src == None or src == 'mongo' or src == 'local' or src == 'sftp'), \
'error, src must be one of None,local,mongo,sftp'
if(eTime == None):
eTime = sTime+dt.timedelta(days=1)
#create a datapointer object
myPtr = radDataPtr(sTime=sTime,eTime=eTime,stid=int(network().getRadarByCode(rad).id),
channel=channel,bmnum=bmnum,cp=cp)
filelist = []
if(fileType == 'fitex'): arr = ['fitex','fitacf','lmfit']
elif(fileType == 'fitacf'): arr = ['fitacf','fitex','lmfit']
elif(fileType == 'lmfit'): arr = ['lmfit','fitex','fitacf']
else: arr = [fileType]
#move back a little in time because files often start at 2 mins after the hour
sTime = sTime-dt.timedelta(minutes=4)
#a temporary directory to store a temporary file
tmpDir = '/tmp/fit/'
d = os.path.dirname(tmpDir)
if not os.path.exists(d):
os.makedirs(d)
#FIRST, check if a specific filename was given
if(fileName != None):
try:
if(not os.path.isfile(fileName)):
print 'problem reading',fileName,':file does not exist'
return None
outname = tmpDir+str(int(datetimeToEpoch(dt.datetime.now())))
if(string.find(fileName,'.bz2') != -1):
outname = string.replace(fileName,'.bz2','')
print 'bunzip2 -c '+fileName+' > '+outname+'\n'
os.system('bunzip2 -c '+fileName+' > '+outname)
elif(string.find(fileName,'.gz') != -1):
outname = string.replace(fileName,'.gz','')
print 'gunzip -c '+fileName+' > '+outname+'\n'
os.system('gunzip -c '+fileName+' > '+outname)
else:
os.system('cp '+fileName+' '+outname)
print 'cp '+fileName+' '+outname
filelist.append(outname)
myPtr.fType,myPtr.dType = custType,'dmap'
except Exception, e:
print e
print 'problem reading file',fileName
return None
