def __init__(self,sTime=None,radcode=None,eTime=None,stid=None,channel=None,bmnum=None,cp=None, \ fileType=None,filtered=False, src=None,fileName=None,noCache=False): import datetime as dt import os,glob,string from pydarn.radar import network self.sTime = sTime self.eTime = eTime self.stid = stid self.channel = channel self.bmnum = bmnum self.cp = cp self.fType = fileType self.dType = None self.fBeam = None self.recordIndex = None self.scanStartIndex = None self.__filename = fileName self.__filtered = filtered self.__nocache = noCache self.__src = src self.__fd = None self.__ptr = None #check inputs assert(isinstance(sTime,dt.datetime)), \ 'error, sTime must be datetime object' assert(eTime == None or isinstance(eTime,dt.datetime)), \ 'error, eTime must be datetime object or None' assert(channel == None or (isinstance(channel,str) and len(channel) == 1)), \ 'error, channel must be None or a 1-letter string' assert(bmnum == None or isinstance(bmnum,int)), \ 'error, bmnum must be an int or None' assert(cp == None or isinstance(cp,int)), \ 'error, cp must be an int or None' assert(fileType == 'rawacf' or fileType == 'fitacf' or \ fileType == 'fitex' or fileType == 'lmfit' or fileType == 'iqdat'), \ 'error, fileType must be one of: rawacf,fitacf,fitex,lmfit,iqdat' assert(fileName == None or isinstance(fileName,str)), \ 'error, fileName must be None or a string' assert(isinstance(filtered,bool)), \ 'error, filtered must be True of False' assert(src == None or src == 'local' or src == 'sftp'), \ 'error, src must be one of None,local,sftp' if radcode is not None: assert(isinstance(radcode,str)), \ 'error, radcode must be None or a string' segments=radcode.split(".") try: rad=segments[0] except: rad=None try: chan=segments[1] except: chan=None assert(isinstance(rad,str) and len(rad) == 3), \ 'error, rad must be a 3 char string' self.stid=int(network().getRadarByCode(rad).id) if(self.eTime == None): self.eTime = self.sTime+dt.timedelta(days=1) filelist = [] if(fileType == 'fitex'): arr = ['fitex','fitacf','lmfit'] elif(fileType == 'fitacf'): arr = ['fitacf','fitex','lmfit'] elif(fileType == 'lmfit'): arr = ['lmfit','fitex','fitacf'] else: arr = [fileType] #move back a little in time because files often start at 2 mins after the hour sTime = sTime-dt.timedelta(minutes=4) #a temporary directory to store a temporary file try: tmpDir=os.environ['DAVIT_TMPDIR'] except: tmpDir = '/tmp/sd/' d = os.path.dirname(tmpDir) if not os.path.exists(d): os.makedirs(d) cached = False fileSt = None #FIRST, check if a specific filename was given if fileName != None: try: if(not os.path.isfile(fileName)): print 'problem reading',fileName,':file does not exist' return None outname = tmpDir+str(int(datetimeToEpoch(dt.datetime.now()))) if(string.find(fileName,'.bz2') != -1): outname = string.replace(fileName,'.bz2','') print 'bunzip2 -c '+fileName+' > '+outname+'\n' os.system('bunzip2 -c '+fileName+' > '+outname) elif(string.find(fileName,'.gz') != -1): outname = string.replace(fileName,'.gz','') print 'gunzip -c '+fileName+' > '+outname+'\n' os.system('gunzip -c '+fileName+' > '+outname) else: os.system('cp '+fileName+' '+outname) print 'cp '+fileName+' '+outname filelist.append(outname) self.dType = 'dmap' fileSt = sTime except Exception, e: print e print 'problem reading file',fileName return None
def overlayFov(Basemap, codes=None, ids=None, names=None, dateTime=None, all=False, maxGate=None, rangeLimits=None, model='IS', fovColor=None, fovAlpha=0.2, beams=None, beamsColors=None, hemi=None, fovObj=None, zorder=2, lineColor='k', lineWidth=1): """Overlay FoV position(s) on map **Args**: * **Basemap**: a python Basemap object on which to overplot the radar position(s) * **[codes]**: a list of radar 3-letter codes to plot * **[ids]**: a list of radar IDs to plot * **[names]**: a list of radar names to plot * **[dateTime]**: the date and time as a python datetime object * **[all]**: set to true to plot all the radars (active ones) * **[maxGate]**: Maximum number of gates to be plotted. Defaults to hdw.dat information. * **[rangeLimits]**: (2-element list) Plot only between the range gates specified. * **model**: * **'IS'**: for ionopsheric scatter projection model (default) * **'GS'**: for ground scatter projection model * **None**: if you are really confident in your elevation or altitude values * **[zorder]**: the overlay order number * **[lineColor]**: FoV contour line color * **[lineWidth]**: FoV contour line width * **[fovColor]**: field of view fill color * **[fovAlpha]**: field of view fill color transparency * **[fovObj]**: a fov object. See pydarn.radar.radFov.fov * **[hemi]**: 'north' or 'south', ignore radars from the other hemisphere * **[beams]**: hightlight specified beams * **[beamsColors]**: colors of the hightlighted beams **Returns**: * None **Example**: :: import pydarn, utils width = 111e3*40 m = utils.plotUtils.mapObj(width=width, height=width, lat_0=50., lon_0=-95.) # Plot radar position and code pydarn.plot.overlayRadar(m, fontSize=12, codes='bks') # Plot radar fov pydarn.plot.overlayFov(m, codes='bks', maxGate=75, beams=[0,4,7,8,23]) written by Sebastien, 2012-09 """ from pydarn.radar import network from pydarn.radar.radFov import fov from datetime import datetime as dt from datetime import timedelta import matplotlib.cm as cm from numpy import transpose, ones, concatenate, vstack, shape from matplotlib.patches import Polygon from pylab import gca # Set default date/time to now if not dateTime: dateTime = dt.utcnow() # Load radar structure NetworkObj = network() # If all radars are to be plotted, create the list if all: codes = NetworkObj.getAllCodes(datetime=dateTime, hemi=hemi) # Define how the radars to be plotted are identified (code, id or name) if codes: input = {'meth': 'code', 'vals': codes} elif ids: input = {'meth': 'id', 'vals': ids} elif names: input = {'meth': 'name', 'vals': names} else: print 'overlayFov: no radars to plot' return # Check if radars is given as a list if not isinstance(input['vals'], list): input['vals'] = [input['vals']] nradars = len(input['vals']) # iterates through radars to be plotted for ir in xrange(nradars): # Get field of view coordinates if (fovObj == None): rad = NetworkObj.getRadarBy(input['vals'][ir], input['meth']) if not rad: continue site = rad.getSiteByDate(dateTime) if not site: continue # Set number of gates to be plotted eGate = site.maxgate - 1 if not maxGate else maxGate if not hasattr(Basemap, 'coords'): radFov = fov(site=site, ngates=eGate + 1, model=model) else: radFov = fov(site=site, ngates=eGate + 1, coords=Basemap.coords, model=model) else: radFov = fovObj eGate = len(fovObj.gates) if rangeLimits is not None: sGate = rangeLimits[0] eGate = rangeLimits[1] else: sGate = 0 # Get radar coordinates in map projection if hasattr(Basemap, 'coords'): x, y = Basemap(radFov.lonFull, radFov.latFull, coords=radFov.coords) else: x, y = Basemap(radFov.lonFull, radFov.latFull) # Plot field of view # Create contour contourX = concatenate( (x[0, sGate:eGate], x[:, eGate], x[-1, eGate:sGate:-1], x[-1::-1, sGate])) contourY = concatenate( (y[0, sGate:eGate], y[:, eGate], y[-1, eGate:sGate:-1], y[-1::-1, sGate])) # Plot contour Basemap.plot(contourX, contourY, color=lineColor, zorder=zorder, linewidth=lineWidth) # Field of view fill if fovColor: contour = transpose(vstack((contourX, contourY))) patch = Polygon(contour, color=fovColor, alpha=fovAlpha, zorder=zorder) gca().add_patch(patch) # Beams fill if beams: try: [b for b in beams] except: beams = [beams] for ib in beams: if not (0 <= ib <= x.shape[0]): continue if not beamsColors: bColRGB = ib / float(x.shape[0]) bCol = (bColRGB / 2., bColRGB, 1) else: bCol = beamsColors[beams.index(ib)] contourX = concatenate( (x[ib, 0:eGate + 1], x[ib:ib + 2, eGate], x[ib + 1, eGate::-1], x[ib + 1:ib - 1:-1, 0])) contourY = concatenate( (y[ib, 0:eGate + 1], y[ib:ib + 2, eGate], y[ib + 1, eGate::-1], y[ib + 1:ib - 1:-1, 0])) contour = transpose(vstack((contourX, contourY))) patch = Polygon(contour, color=bCol, alpha=.4, zorder=zorder) gca().add_patch(patch) return
def overlayRadar(Basemap, codes=None, ids=None, names=None, dateTime=None, annotate=True, all=False, hemi=None, zorder=2, markerColor='k', markerSize=10, fontSize=10, xOffset=None): """Overlay radar position(s) and name(s) on map **Args**: * **Basemap**: a python Basemap object on which to overplot the radar position(s) * **[codes]**: a list of radar 3-letter codes to plot * **[ids]**: a list of radar IDs to plot * **[names]**: a list of radar names to plot * **[dateTime]**: the date and time as a python datetime object * **[annotate]**: wether or not to show the radar(s) name(s) * **[all]**: set to true to plot all the radars (active ones) * **[hemi]**: 'north' or 'south', ignore radars from the other hemisphere * **[zorder]**: the overlay order number * **[markerColor]**: * **[markerSize]**: [point] * **[fontSize]**: [point] * **[xOffset]**: x-Offset of the annotation in points **Returns**: * None **Example**: :: import pydarn, utils m1 = utils.plotUtils.mapObj(boundinglat=30., gridLabels=True, coords='mag') pydarn.plot.overlayRadar(m1, fontSize=8, all=True, markerSize=5) written by Sebastien, 2012-08 """ from pydarn.radar import network from datetime import datetime as dt from datetime import timedelta from utils.plotUtils import textHighlighted # Set default date/time to now if not dateTime: dateTime = dt.utcnow() # Load radar structure NetworkObj = network() # If all radars are to be plotted, create the list if all: codes = NetworkObj.getAllCodes(datetime=dateTime, hemi=hemi) # Define how the radars to be plotted are identified (code, id or name) if codes: input = {'meth': 'code', 'vals': codes} elif ids: input = {'meth': 'id', 'vals': ids} elif names: input = {'meth': 'name', 'vals': names} else: print 'overlayRadar: no radars to plot' return # Check if radars is given as a list if not isinstance(input['vals'], list): input['vals'] = [input['vals']] # Map width and height width = Basemap.urcrnrx - Basemap.llcrnrx height = Basemap.urcrnry - Basemap.llcrnry if hemi is None: hemiInt = 0 else: hemiInt = 1 if hemi.lower()[0] == 'n' else -1 # iterates through radars to be plotted for radN in input['vals']: rad = NetworkObj.getRadarBy(radN, input['meth']) if not rad: continue site = rad.getSiteByDate(dateTime) if not site: continue # Check for hemisphere specification if site.geolat * hemiInt < 0: continue # Get radar coordinates in map projection if not hasattr(Basemap, 'coords'): x, y = Basemap(site.geolon, site.geolat) else: x, y = Basemap(site.geolon, site.geolat, coords='geo') if not Basemap.xmin <= x <= Basemap.xmax: continue if not Basemap.ymin <= y <= Basemap.ymax: continue # Plot radar position Basemap.scatter(x, y, s=markerSize, marker='o', color=markerColor, zorder=zorder) # Now add radar name if annotate: # If any of the other radar is too close... if rad.code[0] in [ 'adw', 'kod', 'cve', 'fhe', 'wal', 'gbr', 'pyk', 'aze', 'sys' ]: xOff = 5 if not xOffset else xOffset ha = 0 elif rad.code[0] in [ 'ade', 'ksr', 'cvw', 'fhw', 'bks', 'sch', 'sto', 'azw', 'sye' ]: xOff = -5 if not xOffset else xOffset ha = 1 else: xOff = 0.0 ha = .5 # Plot radar name textHighlighted((x, y), rad.code[0].upper(), xytext=(xOff, -5), text_alignment=(ha, 1), variant='small-caps', fontsize=fontSize, zorder=zorder) return
def overlayFov(Basemap, codes=None, ids=None, names=None, dateTime=None, all=False, maxGate=None, fovColor=None, fovAlpha=0.2, beams=None, beamsColors=None, hemi=None, fovObj=None, zorder=2, lineColor='k', lineWidth=1): """Overlay FoV position(s) on map **Args**: * **Basemap**: a python Basemap object on which to overplot the radar position(s) * **[codes]**: a list of radar 3-letter codes to plot * **[ids]**: a list of radar IDs to plot * **[names]**: a list of radar names to plot * **[dateTime]**: the date and time as a python datetime object * **[all]**: set to true to plot all the radars (active ones) * **[maxGate]**: Maximum number of gates to be plotted. Defaults to hdw.dat information. * **[zorder]**: the overlay order number * **[lineColor]**: FoV contour line color * **[lineWidth]**: FoV contour line width * **[fovColor]**: field of view fill color * **[fovAlpha]**: field of view fill color transparency * **[fovObj]**: a fov object. See pydarn.radar.radFov.fov * **[hemi]**: 'north' or 'south', ignore radars from the other hemisphere * **[beams]**: hightlight specified beams * **[beamsColors]**: colors of the hightlighted beams **Returns**: * None **Example**: :: import pydarn, utils width = 111e3*40 m = utils.plotUtils.mapObj(width=width, height=width, lat_0=50., lon_0=-95.) # Plot radar position and code pydarn.plot.overlayRadar(m, fontSize=12, codes='bks') # Plot radar fov pydarn.plot.overlayFov(m, codes='bks', maxGate=75, beams=[0,4,7,8,23]) written by Sebastien, 2012-09 """ from pydarn.radar import network from pydarn.radar.radFov import fov from datetime import datetime as dt from datetime import timedelta import matplotlib.cm as cm from numpy import transpose, ones, concatenate, vstack, shape from matplotlib.patches import Polygon from pylab import gca # Set default date/time to now if not dateTime: dateTime = dt.utcnow() # Load radar structure NetworkObj = network() # If all radars are to be plotted, create the list if all: codes = NetworkObj.getAllCodes(datetime=dateTime, hemi=hemi) # Define how the radars to be plotted are identified (code, id or name) if codes: input = {'meth': 'code', 'vals': codes} elif ids: input = {'meth': 'id', 'vals': ids} elif names: input = {'meth': 'name', 'vals': names} else: print 'overlayFov: no radars to plot' return # Check if radars is given as a list if not isinstance(input['vals'], list): input['vals'] = [input['vals']] nradars = len(input['vals']) # iterates through radars to be plotted for ir in xrange(nradars): # Get field of view coordinates if(fovObj == None): rad = NetworkObj.getRadarBy(input['vals'][ir], input['meth']) if not rad: continue site = rad.getSiteByDate(dateTime) if not site: continue # Set number of gates to be plotted eGate = site.maxgate-1 if not maxGate else maxGate if not hasattr(Basemap, 'coords'): radFov = fov(site=site, ngates=eGate+1) else: radFov = fov(site=site, ngates=eGate+1, coords=Basemap.coords) else: radFov = fovObj eGate = len(fovObj.gates) # Get radar coordinates in map projection if hasattr(Basemap, 'coords'): x, y = Basemap(radFov.lonFull, radFov.latFull, coords=radFov.coords) else: x, y = Basemap(radFov.lonFull, radFov.latFull) # Plot field of view # Create contour contourX = concatenate( (x[0,0:eGate], x[:,eGate], x[-1,eGate::-1], x[-1::-1,0]) ) contourY = concatenate( (y[0,0:eGate], y[:,eGate], y[-1,eGate::-1], y[-1::-1,0]) ) # Plot contour Basemap.plot(contourX, contourY, color=lineColor, zorder=zorder, linewidth=lineWidth) # Field of view fill if fovColor: contour = transpose( vstack((contourX,contourY)) ) patch = Polygon( contour, color=fovColor, alpha=fovAlpha, zorder=zorder) gca().add_patch(patch) # Beams fill if beams: try: [b for b in beams] except: beams = [beams] for ib in beams: if not (0 <= ib <= x.shape[0]): continue if not beamsColors: bColRGB = ib/float(x.shape[0]) bCol = (bColRGB/2.,bColRGB,1) else: bCol = beamsColors[beams.index(ib)] contourX = concatenate( (x[ib,0:eGate+1], x[ib:ib+2,eGate], x[ib+1,eGate::-1], x[ib+1:ib-1:-1,0]) ) contourY = concatenate( (y[ib,0:eGate+1], y[ib:ib+2,eGate], y[ib+1,eGate::-1], y[ib+1:ib-1:-1,0]) ) contour = transpose( vstack((contourX,contourY)) ) patch = Polygon( contour, color=bCol, alpha=.4, zorder=zorder) gca().add_patch(patch) return
def overlayRadar(Basemap, codes=None, ids=None, names=None, dateTime=None, annotate=True, all=False, hemi=None, zorder=2, markerColor='k', markerSize=10, fontSize=10, xOffset=None): """Overlay radar position(s) and name(s) on map **Args**: * **Basemap**: a python Basemap object on which to overplot the radar position(s) * **[codes]**: a list of radar 3-letter codes to plot * **[ids]**: a list of radar IDs to plot * **[names]**: a list of radar names to plot * **[dateTime]**: the date and time as a python datetime object * **[annotate]**: wether or not to show the radar(s) name(s) * **[all]**: set to true to plot all the radars (active ones) * **[hemi]**: 'north' or 'south', ignore radars from the other hemisphere * **[zorder]**: the overlay order number * **[markerColor]**: * **[markerSize]**: [point] * **[fontSize]**: [point] * **[xOffset]**: x-Offset of the annotation in points **Returns**: * None **Example**: :: import pydarn, utils m1 = utils.plotUtils.mapObj(boundinglat=30., gridLabels=True, coords='mag') pydarn.plot.overlayRadar(m1, fontSize=8, all=True, markerSize=5) written by Sebastien, 2012-08 """ from pydarn.radar import network from datetime import datetime as dt from datetime import timedelta from utils.plotUtils import textHighlighted # Set default date/time to now if not dateTime: dateTime = dt.utcnow() # Load radar structure NetworkObj = network() # If all radars are to be plotted, create the list if all: codes = NetworkObj.getAllCodes(datetime=dateTime, hemi=hemi) # Define how the radars to be plotted are identified (code, id or name) if codes: input = {'meth': 'code', 'vals': codes} elif ids: input = {'meth': 'id', 'vals': ids} elif names: input = {'meth': 'name', 'vals': names} else: print 'overlayRadar: no radars to plot' return # Check if radars is given as a list if not isinstance(input['vals'], list): input['vals'] = [input['vals']] # Map width and height width = Basemap.urcrnrx - Basemap.llcrnrx height = Basemap.urcrnry - Basemap.llcrnry if hemi is None: hemiInt = 0 else: hemiInt = 1 if hemi.lower()[0]=='n' else -1 # iterates through radars to be plotted for radN in input['vals']: rad = NetworkObj.getRadarBy(radN, input['meth']) if not rad: continue site = rad.getSiteByDate(dateTime) if not site: continue # Check for hemisphere specification if site.geolat*hemiInt < 0: continue # Get radar coordinates in map projection if not hasattr(Basemap, 'coords'): x,y = Basemap(site.geolon, site.geolat) else: x,y = Basemap(site.geolon, site.geolat, coords='geo') if not Basemap.xmin <= x <= Basemap.xmax: continue if not Basemap.ymin <= y <= Basemap.ymax: continue # Plot radar position Basemap.scatter(x, y, s=markerSize, marker='o', color=markerColor, zorder=zorder) # Now add radar name if annotate: # If any of the other radar is too close... if rad.code[0] in ['adw', 'kod', 'cve', 'fhe', 'wal', 'gbr', 'pyk', 'aze', 'sys']: xOff = 5 if not xOffset else xOffset ha = 0 elif rad.code[0] in ['ade', 'ksr', 'cvw', 'fhw', 'bks', 'sch', 'sto', 'azw', 'sye']: xOff = -5 if not xOffset else xOffset ha = 1 else: xOff = 0.0 ha = .5 # Plot radar name textHighlighted((x, y), rad.code[0].upper(), xytext=(xOff, -5), text_alignment=(ha,1), variant='small-caps', fontsize=fontSize, zorder=zorder) return
def radDataOpen(sTime,radcode,eTime=None,channel=None,bmnum=None,cp=None, \ fileType='fitex',filtered=False, src=None,fileName=None, \ custType='fitex',noCache=False): """A function to establish a pipeline through which we can read radar data. first it tries the mongodb, then it tries to find local files, and lastly it sftp's over to the VT data server. **Args**: * **sTime** (`datetime <http://tinyurl.com/bl352yx>`_): the beginning time for which you want data * **radcode** (str): the 3-letter radar code with optional channel extension for which you want data * **[eTime]** (`datetime <http://tinyurl.com/bl352yx>`_): the last time that you want data for. if this is set to None, it will be set to 1 day after sTime. default = None * **[channel]** (str): the 1-letter code for what channel you want data from, eg 'a','b',... if this is set to None, data from ALL channels will be read. default = None * **[bmnum]** (int): the beam number which you want data for. If this is set to None, data from all beams will be read. default = None * **[cp]** (int): the control program which you want data for. If this is set to None, data from all cp's will be read. default = None * **[fileType]** (str): The type of data you want to read. valid inputs are: 'fitex','fitacf','rawacf','iqdat'. if you choose a fit file format and the specified one isn't found, we will search for one of the others. Beware: if you ask for rawacf/iq data, these files are large and the data transfer might take a long time. default = 'fitex' * **[filtered]** (boolean): a boolean specifying whether you want the fit data to be boxcar filtered. ONLY VALID FOR FIT. default = False * **[src]** (str): the source of the data. valid inputs are 'local' 'sftp'. if this is set to None, it will try all possibilites sequentially. default = None * **[fileName]** (str): the name of a specific file which you want to open. default=None * **[custType]** (str): if fileName is specified, the filetype of the file. default='fitex' * **[noCache]** (boolean): flag to indicate that you do not want to check first for cached files. default = False. **Returns**: * **myPtr** (:class:`pydarn.sdio.radDataTypes.radDataPtr`): a radDataPtr object which contains a link to the data to be read. this can then be passed to radDataReadRec in order to actually read the data. **ENVIRONMENT Variables**: * DAVIT_TMPDIR : Directory used for davitpy temporary file cache. * DAVIT_TMPEXPIRE : Length of time that cached temporary files are valid. After which they will be regenerated. Example: DAVIT_TMPEXPIRE='2h' will reuse temp files in the cache for 2 hours since last access * DAVIT_LOCALDIR : Used to set base directory tree for local file look up * DAVIT_DIRFORMAT : Python string dictionary capable format string appended to local file base directory tree for use with directory structures which encode radar name, channel or date information. Currently supported dictionary keys which can be used: "dirtree" : base directory tree "year" : 0 padded 4 digit year "month" : 0 padded 2 digit month "day" : 0 padded 2 digit day "ftype" : filetype string "radar" : 3-chr radarcode **Example**: :: import datetime as dt myPtr = radDataOpen(dt.datetime(2011,1,1),'bks',eTime=dt.datetime(2011,1,1,2),channel='a', bmnum=7,cp=153,fileType='fitex',filtered=False, src=None): Written by AJ 20130110 """ import paramiko as p import re import string import datetime as dt, os, pydarn.sdio, glob from pydarn.sdio import radDataPtr from pydarn.radar import network from utils.timeUtils import datetimeToEpoch #check inputs assert(isinstance(sTime,dt.datetime)), \ 'error, sTime must be datetime object' segments=radcode.split(".") try: rad=segments[0] except: rad=None try: chan=segments[1] except: chan=None assert(isinstance(rad,str) and len(rad) == 3), \ 'error, rad must be a 3 char string' assert(eTime == None or isinstance(eTime,dt.datetime)), \ 'error, eTime must be datetime object or None' assert(channel == None or (isinstance(channel,str) and len(channel) == 1)), \ 'error, channel must be None or a 1-letter string' assert(bmnum == None or isinstance(bmnum,int)), \ 'error, bmnum must be an int or None' assert(cp == None or isinstance(cp,int)), \ 'error, cp must be an int or None' assert(fileType == 'rawacf' or fileType == 'fitacf' or \ fileType == 'fitex' or fileType == 'iqdat'), \ 'error, fileType must be one of: rawacf,fitacf,fitex,iqdat' assert(fileName == None or isinstance(fileName,str)), \ 'error, fileName must be None or a string' assert(isinstance(filtered,bool)), \ 'error, filtered must be True of False' assert(src == None or src == 'local' or src == 'sftp'), \ 'error, src must be one of None,local,sftp' if(eTime == None): eTime = sTime+dt.timedelta(days=1) #create a datapointer object myPtr = radDataPtr(sTime=sTime,eTime=eTime,stid=int(network().getRadarByCode(rad).id), channel=channel,bmnum=bmnum,cp=cp) filelist = [] # if(fileType == 'fitex'): arr = ['fitex','fitacf','lmfit'] # elif(fileType == 'fitacf'): arr = ['fitacf','fitex','lmfit'] # elif(fileType == 'lmfit'): arr = ['lmfit','fitex','fitacf'] if(fileType == 'fitex'): arr = ['fitex','fitacf'] elif(fileType == 'fitacf'): arr = ['fitacf','fitex'] else: arr = [fileType] #move back a little in time because files often start at 2 mins after the hour sTime = sTime-dt.timedelta(minutes=4) #a temporary directory to store a temporary file try: tmpDir=os.environ['DAVIT_TMPDIR'] except: tmpDir = '/tmp/sd/' d = os.path.dirname(tmpDir) if not os.path.exists(d): os.makedirs(d) cached = False fileSt = None #FIRST, check if a specific filename was given if fileName != None: try: if(not os.path.isfile(fileName)): print 'problem reading',fileName,':file does not exist' return None outname = tmpDir+str(int(datetimeToEpoch(dt.datetime.now()))) if(string.find(fileName,'.bz2') != -1): outname = string.replace(fileName,'.bz2','') print 'bunzip2 -c '+fileName+' > '+outname+'\n' os.system('bunzip2 -c '+fileName+' > '+outname) elif(string.find(fileName,'.gz') != -1): outname = string.replace(fileName,'.gz','') print 'gunzip -c '+fileName+' > '+outname+'\n' os.system('gunzip -c '+fileName+' > '+outname) else: os.system('cp '+fileName+' '+outname) print 'cp '+fileName+' '+outname filelist.append(outname) myPtr.fType,myPtr.dType = custType,'dmap' fileSt = sTime except Exception, e: print e print 'problem reading file',fileName return None
def radDataOpen(sTime,radcode,eTime=None,channel=None,bmnum=None,cp=None, \ fileType='fitex',filtered=False, src=None,fileName=None, \ custType='fitex',noCache=False): """A function to establish a pipeline through which we can read radar data. first it tries the mongodb, then it tries to find local files, and lastly it sftp's over to the VT data server. **Args**: * **sTime** (`datetime <http://tinyurl.com/bl352yx>`_): the beginning time for which you want data * **radcode** (str): the 3-letter radar code with optional channel extension for which you want data * **[eTime]** (`datetime <http://tinyurl.com/bl352yx>`_): the last time that you want data for. if this is set to None, it will be set to 1 day after sTime. default = None * **[channel]** (str): the 1-letter code for what channel you want data from, eg 'a','b',... if this is set to None, data from ALL channels will be read. default = None * **[bmnum]** (int): the beam number which you want data for. If this is set to None, data from all beams will be read. default = None * **[cp]** (int): the control program which you want data for. If this is set to None, data from all cp's will be read. default = None * **[fileType]** (str): The type of data you want to read. valid inputs are: 'fitex','fitacf','rawacf','iqdat'. if you choose a fit file format and the specified one isn't found, we will search for one of the others. Beware: if you ask for rawacf/iq data, these files are large and the data transfer might take a long time. default = 'fitex' * **[filtered]** (boolean): a boolean specifying whether you want the fit data to be boxcar filtered. ONLY VALID FOR FIT. default = False * **[src]** (str): the source of the data. valid inputs are 'local' 'sftp'. if this is set to None, it will try all possibilites sequentially. default = None * **[fileName]** (str): the name of a specific file which you want to open. default=None * **[custType]** (str): if fileName is specified, the filetype of the file. default='fitex' * **[noCache]** (boolean): flag to indicate that you do not want to check first for cached files. default = False. **Returns**: * **myPtr** (:class:`pydarn.sdio.radDataTypes.radDataPtr`): a radDataPtr object which contains a link to the data to be read. this can then be passed to radDataReadRec in order to actually read the data. **ENVIRONMENT Variables**: * DAVIT_TMPDIR : Directory used for davitpy temporary file cache. * DAVIT_TMPEXPIRE : Length of time that cached temporary files are valid. After which they will be regenerated. Example: DAVIT_TMPEXPIRE='2h' will reuse temp files in the cache for 2 hours since last access * DAVIT_LOCALDIR : Used to set base directory tree for local file look up * DAVIT_DIRFORMAT : Python string dictionary capable format string appended to local file base directory tree for use with directory structures which encode radar name, channel or date information. Currently supported dictionary keys which can be used: "dirtree" : base directory tree "year" : 0 padded 4 digit year "month" : 0 padded 2 digit month "day" : 0 padded 2 digit day "ftype" : filetype string "radar" : 3-chr radarcode **Example**: :: import datetime as dt myPtr = radDataOpen(dt.datetime(2011,1,1),'bks',eTime=dt.datetime(2011,1,1,2),channel='a', bmnum=7,cp=153,fileType='fitex',filtered=False, src=None): Written by AJ 20130110 """ import paramiko as p import re import string import datetime as dt, os, pydarn.sdio, glob from pydarn.sdio import radDataPtr from pydarn.radar import network from utils.timeUtils import datetimeToEpoch #check inputs assert(isinstance(sTime,dt.datetime)), \ 'error, sTime must be datetime object' segments = radcode.split(".") try: rad = segments[0] except: rad = None try: chan = segments[1] except: chan = None assert(isinstance(rad,str) and len(rad) == 3), \ 'error, rad must be a 3 char string' assert(eTime == None or isinstance(eTime,dt.datetime)), \ 'error, eTime must be datetime object or None' assert(channel == None or (isinstance(channel,str) and len(channel) == 1)), \ 'error, channel must be None or a 1-letter string' assert(bmnum == None or isinstance(bmnum,int)), \ 'error, bmnum must be an int or None' assert(cp == None or isinstance(cp,int)), \ 'error, cp must be an int or None' assert(fileType == 'rawacf' or fileType == 'fitacf' or \ fileType == 'fitex' or fileType == 'iqdat'), \ 'error, fileType must be one of: rawacf,fitacf,fitex,iqdat' assert(fileName == None or isinstance(fileName,str)), \ 'error, fileName must be None or a string' assert(isinstance(filtered,bool)), \ 'error, filtered must be True of False' assert(src == None or src == 'local' or src == 'sftp'), \ 'error, src must be one of None,local,sftp' if (eTime == None): eTime = sTime + dt.timedelta(days=1) #create a datapointer object myPtr = radDataPtr(sTime=sTime, eTime=eTime, stid=int(network().getRadarByCode(rad).id), channel=channel, bmnum=bmnum, cp=cp) filelist = [] # if(fileType == 'fitex'): arr = ['fitex','fitacf','lmfit'] # elif(fileType == 'fitacf'): arr = ['fitacf','fitex','lmfit'] # elif(fileType == 'lmfit'): arr = ['lmfit','fitex','fitacf'] if (fileType == 'fitex'): arr = ['fitex', 'fitacf'] elif (fileType == 'fitacf'): arr = ['fitacf', 'fitex'] else: arr = [fileType] #move back a little in time because files often start at 2 mins after the hour sTime = sTime - dt.timedelta(minutes=4) #a temporary directory to store a temporary file try: tmpDir = os.environ['DAVIT_TMPDIR'] except: tmpDir = '/tmp/sd/' d = os.path.dirname(tmpDir) if not os.path.exists(d): os.makedirs(d) cached = False fileSt = None #FIRST, check if a specific filename was given if fileName != None: try: if (not os.path.isfile(fileName)): print 'problem reading', fileName, ':file does not exist' return None outname = tmpDir + str(int(datetimeToEpoch(dt.datetime.now()))) if (string.find(fileName, '.bz2') != -1): outname = string.replace(fileName, '.bz2', '') print 'bunzip2 -c ' + fileName + ' > ' + outname + '\n' os.system('bunzip2 -c ' + fileName + ' > ' + outname) elif (string.find(fileName, '.gz') != -1): outname = string.replace(fileName, '.gz', '') print 'gunzip -c ' + fileName + ' > ' + outname + '\n' os.system('gunzip -c ' + fileName + ' > ' + outname) else: os.system('cp ' + fileName + ' ' + outname) print 'cp ' + fileName + ' ' + outname filelist.append(outname) myPtr.fType, myPtr.dType = custType, 'dmap' fileSt = sTime except Exception, e: print e print 'problem reading file', fileName return None
def plot(date=None, traceA=None, traceB=None, saveDb=True, noPlot=False): """ Plot RBSP footprints """ assert not date == None or not ( traceA == None and traceB == None), 'Either give a date, or a trace for A and B' datapath = '/home/sebastien/Documents/code/rbsp/data/' imgpath = '/home/sebastien/Documents/code/rbsp/img/' # Read trace if not traceA: traceA = ts.tsygTrace(filename=datapath + 'trace.{}.A.dat'.format(date.strftime('%Y%m%d'))) if not traceB: traceB = ts.tsygTrace(filename=datapath + 'trace.{}.B.dat'.format(date.strftime('%Y%m%d'))) if not date: date = traceA.datetime[0] if not noPlot: # Init figure fig = pylab.figure(figsize=(11, 6.5)) pylab.rcParams.update({'font.size': 14}) fovCol = (.5, .6, .6) fovAlpha = .2 # NH ax1 = fig.add_subplot(121) m1 = Basemap(projection='npstere', boundinglat=35, lon_0=270, resolution='l') m1.drawcoastlines(color='.5') m1.fillcontinents(color=(.8, .8, .8)) # draw parallels and meridians. m1.drawparallels(np.arange(-80., 81., 20.), color='.6') m1.drawmeridians(np.arange(-180., 181., 20.), color='.6') m1.drawmapboundary() # Draw FP x, y = m1(traceA.lonNH, traceA.latNH) m1.scatter(x, y, color='r', zorder=3, s=5) x, y = m1(traceB.lonNH, traceB.latNH) m1.scatter(x, y, color='b', zorder=3, s=5) overlayFov(m1, all=True, hemi='north', maxGate=75, dateTime=traceA.datetime[0], fovColor=fovCol, lineColor=fovCol, lineWidth=0, fovAlpha=fovAlpha) # SH ax2 = fig.add_subplot(122) m2 = Basemap(projection='spstere', boundinglat=-35, lon_0=270, resolution='l') m2.drawcoastlines(color='.5') m2.fillcontinents(color=(.8, .8, .8)) # draw parallels and meridians. m2.drawparallels(np.arange(-80., 81., 20.), color='.6') m2.drawmeridians(np.arange(-180., 181., 20.), color='.6') m2.drawmapboundary() # Draw FP x, y = m2(traceA.lonSH, traceA.latSH) m2.scatter(x, y, color='r', zorder=3, s=5) x, y = m2(traceB.lonSH, traceB.latSH) m2.scatter(x, y, color='b', zorder=3, s=5) overlayFov(m2, all=True, hemi='south', maxGate=75, dateTime=traceA.datetime[0], fovColor=fovCol, lineColor=fovCol, lineWidth=0, fovAlpha=fovAlpha) fig.tight_layout() Title = '{}'.format(date.strftime('%Y - %b - %d')) fig.text(.02, .92, Title) fig.text(.02, .09, 'VAP-A', color='r', va='top') fig.text(.1, .09, 'VAP-B', color='b', va='top') Title = '{}'.format(date.strftime('%Y - %b - %d')) fig.text(.51, .92, Title) fig.text(.51, .09, 'VAP-A', color='r', va='top') fig.text(.59, .09, 'VAP-B', color='b', va='top') # List apogees minsA = np.r_[True, traceA.rho[1:] >= traceA.rho[:-1]] & np.r_[ traceA.rho[:-1] > traceA.rho[1:], True] minsA[0] = minsA[-1] = False lonANH = traceA.lonNH[minsA] latANH = traceA.latNH[minsA] lonASH = traceA.lonSH[minsA] latASH = traceA.latSH[minsA] timeA = traceA.datetime[minsA] minsB = np.r_[True, traceB.rho[1:] >= traceB.rho[:-1]] & np.r_[ traceB.rho[:-1] > traceB.rho[1:], True] minsB[0] = minsB[-1] = False lonBNH = traceB.lonNH[minsB] latBNH = traceB.latNH[minsB] lonBSH = traceB.lonSH[minsB] latBSH = traceB.latSH[minsB] timeB = traceB.datetime[minsB] rads = network() apoA_NH = '' apoA_SH = '' radsANH = [] radsASH = [] for i in xrange(len(lonANH)): apoA_NH += '({}UT, {:4.2f}E, {:4.2f}N) | '.format( timeA[i].strftime('%H:%M'), lonANH[i], latANH[i]) tradANH = rads.getRadarsByPosition(latANH[i], lonANH[i], 300., datetime=timeA[i]) if not noPlot: x, y = m1(lonANH[i], latANH[i]) # ax1.text(x, y, timeA[i].strftime('%H:%M'), zorder=5, clip_on=True, color=(.3,0,0), fontsize=10, ha='center') textHighlighted((x, y), timeA[i].strftime('%H:%M'), ax=ax1, zorder=6, color=(.3, 0, 0), fontsize=10) if tradANH: radsANH.append(tradANH) if saveDb: writeToDb(timeA[i], latANH[i], lonANH[i], 'A', tradANH) apoA_SH += '({}UT, {:4.2f}E, {:4.2f}N) | '.format( timeA[i].strftime('%H:%M'), lonASH[i], latASH[i]) tradASH = rads.getRadarsByPosition(latASH[i], lonASH[i], 300., datetime=timeA[i]) if not noPlot: x, y = m2(lonASH[i], latASH[i]) # ax2.text(x, y, timeA[i].strftime('%H:%M'), zorder=5, clip_on=True, color=(.3,0,0), fontsize=10, ha='center') textHighlighted((x, y), timeA[i].strftime('%H:%M'), ax=ax2, zorder=6, color=(.3, 0, 0), fontsize=10) if tradASH: radsASH.append(tradASH) if saveDb: writeToDb(timeA[i], latASH[i], lonASH[i], 'A', tradASH) apoB_NH = '' apoB_SH = '' radsBNH = [] radsBSH = [] for i in xrange(len(lonBNH)): apoB_NH += '({}UT, {:4.2f}E, {:4.2f}N) '.format( timeB[i].strftime('%H:%M'), lonBNH[i], latBNH[i]) tradBNH = rads.getRadarsByPosition(latBNH[i], lonBNH[i], 300., datetime=timeB[i]) if not noPlot: x, y = m1(lonBNH[i], latBNH[i]) # ax1.text(x, y, timeB[i].strftime('%H:%M'), zorder=5, clip_on=True, color=(0,0,.3), fontsize=10, ha='center') textHighlighted((x, y), timeB[i].strftime('%H:%M'), ax=ax1, zorder=6, color=(0, 0, .3), fontsize=10) if tradBNH: radsBNH.append(tradBNH) if saveDb: writeToDb(timeB[i], latBNH[i], lonBNH[i], 'B', tradBNH) apoB_SH += '({}UT, {:4.2f}E, {:4.2f}N) '.format( timeB[i].strftime('%H:%M'), lonBSH[i], latBSH[i]) tradBSH = rads.getRadarsByPosition(latBSH[i], lonBSH[i], 300., datetime=timeB[i]) if not noPlot: x, y = m2(lonBSH[i], latBSH[i]) # ax2.text(x, y, timeB[i].strftime('%H:%M'), zorder=5, clip_on=True, color=(0,0,.3), fontsize=10, ha='center') textHighlighted((x, y), timeB[i].strftime('%H:%M'), ax=ax2, zorder=6, color=(0, 0, .3), fontsize=10) if tradBSH: radsBSH.append(tradBSH) if saveDb: writeToDb(timeB[i], latBSH[i], lonBSH[i], 'B', tradBSH) if not noPlot: fig.savefig(imgpath + 'rbsp.map.{}.pdf'.format(date.strftime('%Y%m%d'))) fig.savefig(imgpath + 'rbsp.map.{}.svg'.format(date.strftime('%Y%m%d'))) if __name__ == '__main__': fig.clf() pylab.close(fig)
def plot(date=None, traceA=None, traceB=None, saveDb=True, noPlot=False): """ Plot RBSP footprints """ assert not date==None or not (traceA==None and traceB==None), 'Either give a date, or a trace for A and B' datapath = '/home/sebastien/Documents/code/rbsp/data/' imgpath = '/home/sebastien/Documents/code/rbsp/img/' # Read trace if not traceA: traceA = ts.tsygTrace(filename=datapath+'trace.{}.A.dat'.format(date.strftime('%Y%m%d'))) if not traceB: traceB = ts.tsygTrace(filename=datapath+'trace.{}.B.dat'.format(date.strftime('%Y%m%d'))) if not date: date = traceA.datetime[0] if not noPlot: # Init figure fig = pylab.figure(figsize=(11,6.5)) pylab.rcParams.update({'font.size': 14}) fovCol = (.5,.6,.6) fovAlpha = .2 # NH ax1 = fig.add_subplot(121) m1 = Basemap(projection='npstere',boundinglat=35,lon_0=270,resolution='l') m1.drawcoastlines(color='.5') m1.fillcontinents(color=(.8,.8,.8)) # draw parallels and meridians. m1.drawparallels(np.arange(-80.,81.,20.), color='.6') m1.drawmeridians(np.arange(-180.,181.,20.), color='.6') m1.drawmapboundary() # Draw FP x, y = m1(traceA.lonNH, traceA.latNH) m1.scatter(x, y, color='r', zorder=3, s=5) x, y = m1(traceB.lonNH, traceB.latNH) m1.scatter(x, y, color='b', zorder=3, s=5) overlayFov(m1, all=True, hemi='north', maxGate=75, dateTime=traceA.datetime[0], fovColor=fovCol, lineColor=fovCol, lineWidth=0, fovAlpha=fovAlpha) # SH ax2 = fig.add_subplot(122) m2 = Basemap(projection='spstere',boundinglat=-35,lon_0=270,resolution='l') m2.drawcoastlines(color='.5') m2.fillcontinents(color=(.8,.8,.8)) # draw parallels and meridians. m2.drawparallels(np.arange(-80.,81.,20.), color='.6') m2.drawmeridians(np.arange(-180.,181.,20.), color='.6') m2.drawmapboundary() # Draw FP x, y = m2(traceA.lonSH, traceA.latSH) m2.scatter(x, y, color='r', zorder=3, s=5) x, y = m2(traceB.lonSH, traceB.latSH) m2.scatter(x, y, color='b', zorder=3, s=5) overlayFov(m2, all=True, hemi='south', maxGate=75, dateTime=traceA.datetime[0], fovColor=fovCol, lineColor=fovCol, lineWidth=0, fovAlpha=fovAlpha) fig.tight_layout() Title = '{}'.format(date.strftime('%Y - %b - %d')) fig.text(.02, .92, Title) fig.text(.02, .09, 'VAP-A', color='r', va='top') fig.text(.1, .09, 'VAP-B', color='b', va='top') Title = '{}'.format(date.strftime('%Y - %b - %d')) fig.text(.51, .92, Title) fig.text(.51, .09, 'VAP-A', color='r', va='top') fig.text(.59, .09, 'VAP-B', color='b', va='top') # List apogees minsA = np.r_[True, traceA.rho[1:] >= traceA.rho[:-1]] & np.r_[traceA.rho[:-1] > traceA.rho[1:], True] minsA[0] = minsA[-1] = False lonANH = traceA.lonNH[minsA] latANH = traceA.latNH[minsA] lonASH = traceA.lonSH[minsA] latASH = traceA.latSH[minsA] timeA = traceA.datetime[minsA] minsB = np.r_[True, traceB.rho[1:] >= traceB.rho[:-1]] & np.r_[traceB.rho[:-1] > traceB.rho[1:], True] minsB[0] = minsB[-1] = False lonBNH = traceB.lonNH[minsB] latBNH = traceB.latNH[minsB] lonBSH = traceB.lonSH[minsB] latBSH = traceB.latSH[minsB] timeB = traceB.datetime[minsB] rads = network() apoA_NH = '' apoA_SH = '' radsANH = [] radsASH = [] for i in xrange(len(lonANH)): apoA_NH += '({}UT, {:4.2f}E, {:4.2f}N) | '.format(timeA[i].strftime('%H:%M'), lonANH[i], latANH[i]) tradANH = rads.getRadarsByPosition(latANH[i], lonANH[i], 300., datetime=timeA[i]) if not noPlot: x, y = m1(lonANH[i], latANH[i]) # ax1.text(x, y, timeA[i].strftime('%H:%M'), zorder=5, clip_on=True, color=(.3,0,0), fontsize=10, ha='center') textHighlighted((x,y), timeA[i].strftime('%H:%M'), ax=ax1, zorder=6, color=(.3,0,0), fontsize=10) if tradANH: radsANH.append( tradANH ) if saveDb: writeToDb(timeA[i], latANH[i], lonANH[i], 'A', tradANH) apoA_SH += '({}UT, {:4.2f}E, {:4.2f}N) | '.format(timeA[i].strftime('%H:%M'), lonASH[i], latASH[i]) tradASH = rads.getRadarsByPosition(latASH[i], lonASH[i], 300., datetime=timeA[i]) if not noPlot: x, y = m2(lonASH[i], latASH[i]) # ax2.text(x, y, timeA[i].strftime('%H:%M'), zorder=5, clip_on=True, color=(.3,0,0), fontsize=10, ha='center') textHighlighted((x,y), timeA[i].strftime('%H:%M'), ax=ax2, zorder=6, color=(.3,0,0), fontsize=10) if tradASH: radsASH.append( tradASH ) if saveDb: writeToDb(timeA[i], latASH[i], lonASH[i], 'A', tradASH) apoB_NH = '' apoB_SH = '' radsBNH = [] radsBSH = [] for i in xrange(len(lonBNH)): apoB_NH += '({}UT, {:4.2f}E, {:4.2f}N) '.format(timeB[i].strftime('%H:%M'), lonBNH[i], latBNH[i]) tradBNH = rads.getRadarsByPosition(latBNH[i], lonBNH[i], 300., datetime=timeB[i]) if not noPlot: x, y = m1(lonBNH[i], latBNH[i]) # ax1.text(x, y, timeB[i].strftime('%H:%M'), zorder=5, clip_on=True, color=(0,0,.3), fontsize=10, ha='center') textHighlighted((x,y), timeB[i].strftime('%H:%M'), ax=ax1, zorder=6, color=(0,0,.3), fontsize=10) if tradBNH: radsBNH.append( tradBNH ) if saveDb: writeToDb(timeB[i], latBNH[i], lonBNH[i], 'B', tradBNH) apoB_SH += '({}UT, {:4.2f}E, {:4.2f}N) '.format(timeB[i].strftime('%H:%M'), lonBSH[i], latBSH[i]) tradBSH = rads.getRadarsByPosition(latBSH[i], lonBSH[i], 300., datetime=timeB[i]) if not noPlot: x, y = m2(lonBSH[i], latBSH[i]) # ax2.text(x, y, timeB[i].strftime('%H:%M'), zorder=5, clip_on=True, color=(0,0,.3), fontsize=10, ha='center') textHighlighted((x,y), timeB[i].strftime('%H:%M'), ax=ax2, zorder=6, color=(0,0,.3), fontsize=10) if tradBSH: radsBSH.append( tradBSH ) if saveDb: writeToDb(timeB[i], latBSH[i], lonBSH[i], 'B', tradBSH) if not noPlot: fig.savefig( imgpath+'rbsp.map.{}.pdf'.format(date.strftime('%Y%m%d')) ) fig.savefig( imgpath+'rbsp.map.{}.svg'.format(date.strftime('%Y%m%d')) ) if __name__ == '__main__': fig.clf() pylab.close(fig)
def radDataOpen(sTime,rad,eTime=None,channel=None,bmnum=None,cp=None, \ fileType='fitex',filtered=False, src=None,fileName=None, \ custType='fitex'): """A function to establish a pipeline through which we can read radar data. first it tries the mongodb, then it tries to find local files, and lastly it sftp's over to the VT data server. **Args**: * **sTime** (`datetime <http://tinyurl.com/bl352yx>`_): the beginning time for which you want data * **rad** (str): the 3-letter radar code for which you want data * **[eTime]** (`datetime <http://tinyurl.com/bl352yx>`_): the last time that you want data for. if this is set to None, it will be set to 1 day after sTime. default = None * **[channel]** (str): the 1-letter code for what channel you want data from, eg 'a','b',... if this is set to None, data from ALL channels will be read. default = None * **[bmnum]** (int): the beam number which you want data for. If this is set to None, data from all beams will be read. default = None * **[cp]** (int): the control program which you want data for. If this is set to None, data from all cp's will be read. default = None * **[fileType]** (str): The type of data you want to read. valid inputs are: 'fitex','fitacf','lmfit','rawacf','iqdat'. if you choose a fit file format and the specified one isn't found, we will search for one of the others. Beware: if you ask for rawacf/iq data, these files are large and the data transfer might take a long time. default = 'fitex' * **[filtered]** (boolean): a boolean specifying whether you want the fit data to be boxcar filtered. ONLY VALID FOR FIT. default = False * **[src]** (str): the source of the data. valid inputs are 'mongo' 'local' 'sftp'. if this is set to None, it will try all possibilites sequentially. default = None * **[fileName]** (str): the name of a specific file which you want to open. default=None * **[custType]** (str): if fileName is specified, the filetype of the file. default='fitex' **Returns**: * **myPtr** (:class:`radDataTypes.radDataPtr`): a radDataPtr object which contains a link to the data to be read. this can then be passed to radDataReadRec in order to actually read the data. **Example**: :: import datetime as dt myPtr = radDataOpen(dt.datetime(2011,1,1),'bks',eTime=dt.datetime(2011,1,1,2),channel='a', bmnum=7,cp=153,fileType='fitex',filtered=False, src=None): Written by AJ 20130110 """ import subprocess as sub, paramiko as p, re, string import datetime as dt, os, pydarn.sdio, glob from pydarn.sdio import radDataPtr from pydarn.radar import network from utils.timeUtils import datetimeToEpoch #check inputs assert(isinstance(sTime,dt.datetime)), \ 'error, sTime must be datetime object' assert(isinstance(rad,str) and len(rad) == 3), \ 'error, rad must be a 3 char string' assert(eTime == None or isinstance(eTime,dt.datetime)), \ 'error, eTime must be datetime object or None' assert(channel == None or (isinstance(channel,str) and len(channel) == 1)), \ 'error, channel must be None or a 1-letter string' assert(bmnum == None or isinstance(bmnum,int)), \ 'error, bmnum must be an int or None' assert(cp == None or isinstance(cp,int)), \ 'error, cp must be an int or None' assert(fileType == 'rawacf' or fileType == 'fitacf' or \ fileType == 'fitex' or fileType == 'lmfit' or fileType == 'iqdat'), \ 'error, fileType must be one of: rawacf,fitacf,fitex,lmfit,iqdat' assert(fileName == None or isinstance(fileName,str)), \ 'error, fileName must be None or a string' assert(isinstance(filtered,bool)), \ 'error, filtered must be True of False' assert(src == None or src == 'mongo' or src == 'local' or src == 'sftp'), \ 'error, src must be one of None,local,mongo,sftp' if(eTime == None): eTime = sTime+dt.timedelta(days=1) #create a datapointer object myPtr = radDataPtr(sTime=sTime,eTime=eTime,stid=int(network().getRadarByCode(rad).id), channel=channel,bmnum=bmnum,cp=cp) filelist = [] if(fileType == 'fitex'): arr = ['fitex','fitacf','lmfit'] elif(fileType == 'fitacf'): arr = ['fitacf','fitex','lmfit'] elif(fileType == 'lmfit'): arr = ['lmfit','fitex','fitacf'] else: arr = [fileType] #move back a little in time because files often start at 2 mins after the hour sTime = sTime-dt.timedelta(minutes=4) #a temporary directory to store a temporary file tmpDir = '/tmp/fit/' d = os.path.dirname(tmpDir) if not os.path.exists(d): os.makedirs(d) #FIRST, check if a specific filename was given if(fileName != None): try: if(not os.path.isfile(fileName)): print 'problem reading',fileName,':file does not exist' return None outname = tmpDir+str(int(datetimeToEpoch(dt.datetime.now()))) if(string.find(fileName,'.bz2') != -1): outname = string.replace(fileName,'.bz2','') print 'bunzip2 -c '+fileName+' > '+outname+'\n' os.system('bunzip2 -c '+fileName+' > '+outname) elif(string.find(fileName,'.gz') != -1): outname = string.replace(fileName,'.gz','') print 'gunzip -c '+fileName+' > '+outname+'\n' os.system('gunzip -c '+fileName+' > '+outname) else: os.system('cp '+fileName+' '+outname) print 'cp '+fileName+' '+outname filelist.append(outname) myPtr.fType,myPtr.dType = custType,'dmap' except Exception, e: print e print 'problem reading file',fileName return None