def sdDataOpen(sTime,hemi='north',eTime=None,fileType='grdex',src=None,fileName=None, \
custType='grdex',noCache=False):
"""A function to establish a pipeline through which we can read radar data. first it tries the mongodb, then it tries to find local files, and lastly it sftp's over to the VT data server.
**Args**:
* **sTime** (`datetime <http://tinyurl.com/bl352yx>`_): the beginning time for which you want data
* **[hemi]** (str): the hemisphere for which you want data, 'north' or 'south'. default = 'north'
* **[eTime]** (`datetime <http://tinyurl.com/bl352yx>`_): the last time that you want data for. if this is set to None, it will be set to 1 day after sTime. default = None
* **[fileType]** (str): The type of data you want to read. valid inputs are: 'grd','grdex','map','mapex'. If you choose a file format and the specified one isn't found, we will search for one of the others (eg mapex instead of map). default = 'grdex'.
* **[src]** (str): the source of the data. valid inputs are 'local' 'sftp'. if this is set to None, it will try all possibilites sequentially. default = None
* **[fileName]** (str): the name of a specific file which you want to open. If this is set, we will not look for cached files. default=None
* **[custType]** (str): if fileName is specified, the filetype of the file. default = 'grdex'
* **[noCache]** (boolean): flag to indicate that you do not want to check first for cached files. default = False.
**Returns**:
* **myPtr** (:class:`pydarn.sdio.sdDataTypes.sdDataPtr`): a sdDataPtr object which contains a link to the data to be read. this can then be passed to sdDataReadRec in order to actually read the data.
**Example**:
::
import datetime as dt
myPtr = sdDataOpen(dt.datetime(2011,1,1),hemi='north'):
Written by AJ 20130607
"""
import paramiko as p
import re
import string
import datetime as dt
import os
import pydarn.sdio
import glob
from pydarn.sdio import sdDataPtr
from pydarn.radar import network
from utils.timeUtils import datetimeToEpoch
#check inputs
assert(isinstance(sTime,dt.datetime)), \
'error, sTime must be datetime object'
assert(hemi == 'north' or hemi == 'south'), \
"error, hemi must be 'north' or 'south'"
assert(eTime == None or isinstance(eTime,dt.datetime)), \
'error, eTime must be datetime object or None'
assert(fileType == 'grd' or fileType == 'grdex' or \
fileType == 'map' or fileType == 'mapex'), \
"error, fileType must be one of: 'grd','grdex','map','mapex'"
assert(fileName == None or isinstance(fileName,str)), \
'error, fileName must be None or a string'
assert(src == None or src == 'local' or src == 'sftp'), \
'error, src must be one of None,local,sftp'
if eTime == None: eTime = sTime+dt.timedelta(days=1)
#create a datapointer object
myPtr = sdDataPtr(sTime=sTime,eTime=eTime,hemi=hemi)
filelist = []
if fileType == 'grd': arr = ['grd','grdex']
elif fileType == 'grdex': arr = ['grdex','grd']
elif fileType == 'map': arr = ['map','mapex']
elif fileType == 'mapex': arr = ['mapex','map']
else: arr = [fileType]
#move back a little in time because files often start at 2 mins after the hour
sTime = sTime-dt.timedelta(minutes=4)
#a temporary directory to store a temporary file
tmpDir = '/tmp/sd/'
d = os.path.dirname(tmpDir)
if not os.path.exists(d):
os.makedirs(d)
cached = False
fileSt = None
#FIRST, check if a specific filename was given
if fileName != None:
try:
if(not os.path.isfile(fileName)):
print 'problem reading',fileName,':file does not exist'
return None
outname = tmpDir+str(int(datetimeToEpoch(dt.datetime.now())))
if(string.find(fileName,'.bz2') != -1):
outname = string.replace(fileName,'.bz2','')
print 'bunzip2 -c '+fileName+' > '+outname+'\n'
os.system('bunzip2 -c '+fileName+' > '+outname)
elif(string.find(fileName,'.gz') != -1):
outname = string.replace(fileName,'.gz','')
print 'gunzip -c '+fileName+' > '+outname+'\n'
os.system('gunzip -c '+fileName+' > '+outname)
else:
os.system('cp '+fileName+' '+outname)
print 'cp '+fileName+' '+outname
filelist.append(outname)
myPtr.fType,myPtr.dType = custType,'dmap'
fileSt = sTime
except Exception, e:
print e
print 'problem reading file',fileName
return None
def radDataOpen(sTime,radcode,eTime=None,channel=None,bmnum=None,cp=None, \
fileType='fitex',filtered=False, src=None,fileName=None, \
custType='fitex',noCache=False):
"""A function to establish a pipeline through which we can read radar data. first it tries the mongodb, then it tries to find local files, and lastly it sftp's over to the VT data server.
**Args**:
* **sTime** (`datetime <http://tinyurl.com/bl352yx>`_): the beginning time for which you want data
* **radcode** (str): the 3-letter radar code with optional channel extension for which you want data
* **[eTime]** (`datetime <http://tinyurl.com/bl352yx>`_): the last time that you want data for. if this is set to None, it will be set to 1 day after sTime. default = None
* **[channel]** (str): the 1-letter code for what channel you want data from, eg 'a','b',... if this is set to None, data from ALL channels will be read. default = None
* **[bmnum]** (int): the beam number which you want data for. If this is set to None, data from all beams will be read. default = None
* **[cp]** (int): the control program which you want data for. If this is set to None, data from all cp's will be read. default = None
* **[fileType]** (str): The type of data you want to read. valid inputs are: 'fitex','fitacf','rawacf','iqdat'. if you choose a fit file format and the specified one isn't found, we will search for one of the others. Beware: if you ask for rawacf/iq data, these files are large and the data transfer might take a long time. default = 'fitex'
* **[filtered]** (boolean): a boolean specifying whether you want the fit data to be boxcar filtered. ONLY VALID FOR FIT. default = False
* **[src]** (str): the source of the data. valid inputs are 'local' 'sftp'. if this is set to None, it will try all possibilites sequentially. default = None
* **[fileName]** (str): the name of a specific file which you want to open. default=None
* **[custType]** (str): if fileName is specified, the filetype of the file. default='fitex'
* **[noCache]** (boolean): flag to indicate that you do not want to check first for cached files. default = False.
**Returns**:
* **myPtr** (:class:`pydarn.sdio.radDataTypes.radDataPtr`): a radDataPtr object which contains a link to the data to be read. this can then be passed to radDataReadRec in order to actually read the data.
**ENVIRONMENT Variables**:
* DAVIT_TMPDIR : Directory used for davitpy temporary file cache.
* DAVIT_TMPEXPIRE : Length of time that cached temporary files are valid. After which they will be regenerated. Example: DAVIT_TMPEXPIRE='2h' will reuse temp files in the cache for 2 hours since last access
* DAVIT_LOCALDIR : Used to set base directory tree for local file look up
* DAVIT_DIRFORMAT : Python string dictionary capable format string appended to local file base directory tree for use with directory structures which encode radar name, channel or date information.
Currently supported dictionary keys which can be used:
"dirtree" : base directory tree
"year" : 0 padded 4 digit year
"month" : 0 padded 2 digit month
"day" : 0 padded 2 digit day
"ftype" : filetype string
"radar" : 3-chr radarcode
**Example**:
::
import datetime as dt
myPtr = radDataOpen(dt.datetime(2011,1,1),'bks',eTime=dt.datetime(2011,1,1,2),channel='a', bmnum=7,cp=153,fileType='fitex',filtered=False, src=None):
Written by AJ 20130110
"""
import paramiko as p
import re
import string
import datetime as dt, os, pydarn.sdio, glob
from pydarn.sdio import radDataPtr
from pydarn.radar import network
from utils.timeUtils import datetimeToEpoch
#check inputs
assert(isinstance(sTime,dt.datetime)), \
'error, sTime must be datetime object'
segments=radcode.split(".")
try: rad=segments[0]
except: rad=None
try: chan=segments[1]
except: chan=None
assert(isinstance(rad,str) and len(rad) == 3), \
'error, rad must be a 3 char string'
assert(eTime == None or isinstance(eTime,dt.datetime)), \
'error, eTime must be datetime object or None'
assert(channel == None or (isinstance(channel,str) and len(channel) == 1)), \
'error, channel must be None or a 1-letter string'
assert(bmnum == None or isinstance(bmnum,int)), \
'error, bmnum must be an int or None'
assert(cp == None or isinstance(cp,int)), \
'error, cp must be an int or None'
assert(fileType == 'rawacf' or fileType == 'fitacf' or \
fileType == 'fitex' or fileType == 'iqdat'), \
'error, fileType must be one of: rawacf,fitacf,fitex,iqdat'
assert(fileName == None or isinstance(fileName,str)), \
'error, fileName must be None or a string'
assert(isinstance(filtered,bool)), \
'error, filtered must be True of False'
assert(src == None or src == 'local' or src == 'sftp'), \
'error, src must be one of None,local,sftp'
if(eTime == None):
eTime = sTime+dt.timedelta(days=1)
#create a datapointer object
myPtr = radDataPtr(sTime=sTime,eTime=eTime,stid=int(network().getRadarByCode(rad).id),
channel=channel,bmnum=bmnum,cp=cp)
filelist = []
# if(fileType == 'fitex'): arr = ['fitex','fitacf','lmfit']
# elif(fileType == 'fitacf'): arr = ['fitacf','fitex','lmfit']
# elif(fileType == 'lmfit'): arr = ['lmfit','fitex','fitacf']
if(fileType == 'fitex'): arr = ['fitex','fitacf']
elif(fileType == 'fitacf'): arr = ['fitacf','fitex']
else: arr = [fileType]
#move back a little in time because files often start at 2 mins after the hour
sTime = sTime-dt.timedelta(minutes=4)
#a temporary directory to store a temporary file
try:
tmpDir=os.environ['DAVIT_TMPDIR']
except:
tmpDir = '/tmp/sd/'
d = os.path.dirname(tmpDir)
if not os.path.exists(d):
os.makedirs(d)
cached = False
fileSt = None
#FIRST, check if a specific filename was given
if fileName != None:
try:
if(not os.path.isfile(fileName)):
print 'problem reading',fileName,':file does not exist'
return None
outname = tmpDir+str(int(datetimeToEpoch(dt.datetime.now())))
if(string.find(fileName,'.bz2') != -1):
outname = string.replace(fileName,'.bz2','')
print 'bunzip2 -c '+fileName+' > '+outname+'\n'
os.system('bunzip2 -c '+fileName+' > '+outname)
elif(string.find(fileName,'.gz') != -1):
outname = string.replace(fileName,'.gz','')
print 'gunzip -c '+fileName+' > '+outname+'\n'
os.system('gunzip -c '+fileName+' > '+outname)
else:
os.system('cp '+fileName+' '+outname)
print 'cp '+fileName+' '+outname
filelist.append(outname)
myPtr.fType,myPtr.dType = custType,'dmap'
fileSt = sTime
except Exception, e:
print e
print 'problem reading file',fileName
return None
def radDataOpen(sTime,radcode,eTime=None,channel=None,bmnum=None,cp=None, \
fileType='fitex',filtered=False, src=None,fileName=None, \
custType='fitex',noCache=False):
"""A function to establish a pipeline through which we can read radar data. first it tries the mongodb, then it tries to find local files, and lastly it sftp's over to the VT data server.
**Args**:
* **sTime** (`datetime <http://tinyurl.com/bl352yx>`_): the beginning time for which you want data
* **radcode** (str): the 3-letter radar code with optional channel extension for which you want data
* **[eTime]** (`datetime <http://tinyurl.com/bl352yx>`_): the last time that you want data for. if this is set to None, it will be set to 1 day after sTime. default = None
* **[channel]** (str): the 1-letter code for what channel you want data from, eg 'a','b',... if this is set to None, data from ALL channels will be read. default = None
* **[bmnum]** (int): the beam number which you want data for. If this is set to None, data from all beams will be read. default = None
* **[cp]** (int): the control program which you want data for. If this is set to None, data from all cp's will be read. default = None
* **[fileType]** (str): The type of data you want to read. valid inputs are: 'fitex','fitacf','rawacf','iqdat'. if you choose a fit file format and the specified one isn't found, we will search for one of the others. Beware: if you ask for rawacf/iq data, these files are large and the data transfer might take a long time. default = 'fitex'
* **[filtered]** (boolean): a boolean specifying whether you want the fit data to be boxcar filtered. ONLY VALID FOR FIT. default = False
* **[src]** (str): the source of the data. valid inputs are 'local' 'sftp'. if this is set to None, it will try all possibilites sequentially. default = None
* **[fileName]** (str): the name of a specific file which you want to open. default=None
* **[custType]** (str): if fileName is specified, the filetype of the file. default='fitex'
* **[noCache]** (boolean): flag to indicate that you do not want to check first for cached files. default = False.
**Returns**:
* **myPtr** (:class:`pydarn.sdio.radDataTypes.radDataPtr`): a radDataPtr object which contains a link to the data to be read. this can then be passed to radDataReadRec in order to actually read the data.
**ENVIRONMENT Variables**:
* DAVIT_TMPDIR : Directory used for davitpy temporary file cache.
* DAVIT_TMPEXPIRE : Length of time that cached temporary files are valid. After which they will be regenerated. Example: DAVIT_TMPEXPIRE='2h' will reuse temp files in the cache for 2 hours since last access
* DAVIT_LOCALDIR : Used to set base directory tree for local file look up
* DAVIT_DIRFORMAT : Python string dictionary capable format string appended to local file base directory tree for use with directory structures which encode radar name, channel or date information.
Currently supported dictionary keys which can be used:
"dirtree" : base directory tree
"year" : 0 padded 4 digit year
"month" : 0 padded 2 digit month
"day" : 0 padded 2 digit day
"ftype" : filetype string
"radar" : 3-chr radarcode
**Example**:
::
import datetime as dt
myPtr = radDataOpen(dt.datetime(2011,1,1),'bks',eTime=dt.datetime(2011,1,1,2),channel='a', bmnum=7,cp=153,fileType='fitex',filtered=False, src=None):
Written by AJ 20130110
"""
import paramiko as p
import re
import string
import datetime as dt, os, pydarn.sdio, glob
from pydarn.sdio import radDataPtr
from pydarn.radar import network
from utils.timeUtils import datetimeToEpoch
#check inputs
assert(isinstance(sTime,dt.datetime)), \
'error, sTime must be datetime object'
segments = radcode.split(".")
try:
rad = segments[0]
except:
rad = None
try:
chan = segments[1]
except:
chan = None
assert(isinstance(rad,str) and len(rad) == 3), \
'error, rad must be a 3 char string'
assert(eTime == None or isinstance(eTime,dt.datetime)), \
'error, eTime must be datetime object or None'
assert(channel == None or (isinstance(channel,str) and len(channel) == 1)), \
'error, channel must be None or a 1-letter string'
assert(bmnum == None or isinstance(bmnum,int)), \
'error, bmnum must be an int or None'
assert(cp == None or isinstance(cp,int)), \
'error, cp must be an int or None'
assert(fileType == 'rawacf' or fileType == 'fitacf' or \
fileType == 'fitex' or fileType == 'iqdat'), \
'error, fileType must be one of: rawacf,fitacf,fitex,iqdat'
assert(fileName == None or isinstance(fileName,str)), \
'error, fileName must be None or a string'
assert(isinstance(filtered,bool)), \
'error, filtered must be True of False'
assert(src == None or src == 'local' or src == 'sftp'), \
'error, src must be one of None,local,sftp'
if (eTime == None):
eTime = sTime + dt.timedelta(days=1)
#create a datapointer object
myPtr = radDataPtr(sTime=sTime,
eTime=eTime,
stid=int(network().getRadarByCode(rad).id),
channel=channel,
bmnum=bmnum,
cp=cp)
filelist = []
# if(fileType == 'fitex'): arr = ['fitex','fitacf','lmfit']
# elif(fileType == 'fitacf'): arr = ['fitacf','fitex','lmfit']
# elif(fileType == 'lmfit'): arr = ['lmfit','fitex','fitacf']
if (fileType == 'fitex'): arr = ['fitex', 'fitacf']
elif (fileType == 'fitacf'): arr = ['fitacf', 'fitex']
else: arr = [fileType]
#move back a little in time because files often start at 2 mins after the hour
sTime = sTime - dt.timedelta(minutes=4)
#a temporary directory to store a temporary file
try:
tmpDir = os.environ['DAVIT_TMPDIR']
except:
tmpDir = '/tmp/sd/'
d = os.path.dirname(tmpDir)
if not os.path.exists(d):
os.makedirs(d)
cached = False
fileSt = None
#FIRST, check if a specific filename was given
if fileName != None:
try:
if (not os.path.isfile(fileName)):
print 'problem reading', fileName, ':file does not exist'
return None
outname = tmpDir + str(int(datetimeToEpoch(dt.datetime.now())))
if (string.find(fileName, '.bz2') != -1):
outname = string.replace(fileName, '.bz2', '')
print 'bunzip2 -c ' + fileName + ' > ' + outname + '\n'
os.system('bunzip2 -c ' + fileName + ' > ' + outname)
elif (string.find(fileName, '.gz') != -1):
outname = string.replace(fileName, '.gz', '')
print 'gunzip -c ' + fileName + ' > ' + outname + '\n'
os.system('gunzip -c ' + fileName + ' > ' + outname)
else:
os.system('cp ' + fileName + ' ' + outname)
print 'cp ' + fileName + ' ' + outname
filelist.append(outname)
myPtr.fType, myPtr.dType = custType, 'dmap'
fileSt = sTime
except Exception, e:
print e
print 'problem reading file', fileName
return None
def sdDataOpen(sTime,hemi='north',eTime=None,fileType='grdex',src=None,fileName=None, \
custType='grdex',noCache=False):
"""A function to establish a pipeline through which we can read radar data. first it tries the mongodb, then it tries to find local files, and lastly it sftp's over to the VT data server.
**Args**:
* **sTime** (`datetime <http://tinyurl.com/bl352yx>`_): the beginning time for which you want data
* **[hemi]** (str): the hemisphere for which you want data, 'north' or 'south'. default = 'north'
* **[eTime]** (`datetime <http://tinyurl.com/bl352yx>`_): the last time that you want data for. if this is set to None, it will be set to 1 day after sTime. default = None
* **[fileType]** (str): The type of data you want to read. valid inputs are: 'grd','grdex','map','mapex'. If you choose a file format and the specified one isn't found, we will search for one of the others (eg mapex instead of map). default = 'grdex'.
* **[src]** (str): the source of the data. valid inputs are 'local' 'sftp'. if this is set to None, it will try all possibilites sequentially. default = None
* **[fileName]** (str): the name of a specific file which you want to open. If this is set, we will not look for cached files. default=None
* **[custType]** (str): if fileName is specified, the filetype of the file. default = 'grdex'
* **[noCache]** (boolean): flag to indicate that you do not want to check first for cached files. default = False.
**Returns**:
* **myPtr** (:class:`pydarn.sdio.sdDataTypes.sdDataPtr`): a sdDataPtr object which contains a link to the data to be read. this can then be passed to sdDataReadRec in order to actually read the data.
**Example**:
::
import datetime as dt
myPtr = sdDataOpen(dt.datetime(2011,1,1),hemi='north'):
Written by AJ 20130607
"""
import paramiko as p
import re
import string
import datetime as dt
import os
import pydarn.sdio
import glob
from pydarn.sdio import sdDataPtr
from pydarn.radar import network
from utils.timeUtils import datetimeToEpoch
#check inputs
assert(isinstance(sTime,dt.datetime)), \
'error, sTime must be datetime object'
assert(hemi == 'north' or hemi == 'south'), \
"error, hemi must be 'north' or 'south'"
assert(eTime == None or isinstance(eTime,dt.datetime)), \
'error, eTime must be datetime object or None'
assert(fileType == 'grd' or fileType == 'grdex' or \
fileType == 'map' or fileType == 'mapex'), \
"error, fileType must be one of: 'grd','grdex','map','mapex'"
assert(fileName == None or isinstance(fileName,str)), \
'error, fileName must be None or a string'
assert(src == None or src == 'local' or src == 'sftp'), \
'error, src must be one of None,local,sftp'
if eTime == None: eTime = sTime + dt.timedelta(days=1)
#create a datapointer object
myPtr = sdDataPtr(sTime=sTime, eTime=eTime, hemi=hemi)
filelist = []
if fileType == 'grd': arr = ['grd', 'grdex']
elif fileType == 'grdex': arr = ['grdex', 'grd']
elif fileType == 'map': arr = ['map', 'mapex']
elif fileType == 'mapex': arr = ['mapex', 'map']
else: arr = [fileType]
#move back a little in time because files often start at 2 mins after the hour
sTime = sTime - dt.timedelta(minutes=4)
#a temporary directory to store a temporary file
tmpDir = '/tmp/sd/'
d = os.path.dirname(tmpDir)
if not os.path.exists(d):
os.makedirs(d)
cached = False
fileSt = None
#FIRST, check if a specific filename was given
if fileName != None:
try:
if (not os.path.isfile(fileName)):
print 'problem reading', fileName, ':file does not exist'
return None
outname = tmpDir + str(int(datetimeToEpoch(dt.datetime.now())))
if (string.find(fileName, '.bz2') != -1):
outname = string.replace(fileName, '.bz2', '')
print 'bunzip2 -c ' + fileName + ' > ' + outname + '\n'
os.system('bunzip2 -c ' + fileName + ' > ' + outname)
elif (string.find(fileName, '.gz') != -1):
outname = string.replace(fileName, '.gz', '')
print 'gunzip -c ' + fileName + ' > ' + outname + '\n'
os.system('gunzip -c ' + fileName + ' > ' + outname)
else:
os.system('cp ' + fileName + ' ' + outname)
print 'cp ' + fileName + ' ' + outname
filelist.append(outname)
myPtr.fType, myPtr.dType = custType, 'dmap'
fileSt = sTime
except Exception, e:
print e
print 'problem reading file', fileName
return None
def radDataOpen(sTime,rad,eTime=None,channel=None,bmnum=None,cp=None, \
fileType='fitex',filtered=False, src=None,fileName=None, \
custType='fitex'):
"""A function to establish a pipeline through which we can read radar data. first it tries the mongodb, then it tries to find local files, and lastly it sftp's over to the VT data server.
**Args**:
* **sTime** (`datetime <http://tinyurl.com/bl352yx>`_): the beginning time for which you want data
* **rad** (str): the 3-letter radar code for which you want data
* **[eTime]** (`datetime <http://tinyurl.com/bl352yx>`_): the last time that you want data for. if this is set to None, it will be set to 1 day after sTime. default = None
* **[channel]** (str): the 1-letter code for what channel you want data from, eg 'a','b',... if this is set to None, data from ALL channels will be read. default = None
* **[bmnum]** (int): the beam number which you want data for. If this is set to None, data from all beams will be read. default = None
* **[cp]** (int): the control program which you want data for. If this is set to None, data from all cp's will be read. default = None
* **[fileType]** (str): The type of data you want to read. valid inputs are: 'fitex','fitacf','lmfit','rawacf','iqdat'. if you choose a fit file format and the specified one isn't found, we will search for one of the others. Beware: if you ask for rawacf/iq data, these files are large and the data transfer might take a long time. default = 'fitex'
* **[filtered]** (boolean): a boolean specifying whether you want the fit data to be boxcar filtered. ONLY VALID FOR FIT. default = False
* **[src]** (str): the source of the data. valid inputs are 'mongo' 'local' 'sftp'. if this is set to None, it will try all possibilites sequentially. default = None
* **[fileName]** (str): the name of a specific file which you want to open. default=None
* **[custType]** (str): if fileName is specified, the filetype of the file. default='fitex'
**Returns**:
* **myPtr** (:class:`radDataTypes.radDataPtr`): a radDataPtr object which contains a link to the data to be read. this can then be passed to radDataReadRec in order to actually read the data.
**Example**:
::
import datetime as dt
myPtr = radDataOpen(dt.datetime(2011,1,1),'bks',eTime=dt.datetime(2011,1,1,2),channel='a', bmnum=7,cp=153,fileType='fitex',filtered=False, src=None):
Written by AJ 20130110
"""
import subprocess as sub, paramiko as p, re, string
import datetime as dt, os, pydarn.sdio, glob
from pydarn.sdio import radDataPtr
from pydarn.radar import network
from utils.timeUtils import datetimeToEpoch
#check inputs
assert(isinstance(sTime,dt.datetime)), \
'error, sTime must be datetime object'
assert(isinstance(rad,str) and len(rad) == 3), \
'error, rad must be a 3 char string'
assert(eTime == None or isinstance(eTime,dt.datetime)), \
'error, eTime must be datetime object or None'
assert(channel == None or (isinstance(channel,str) and len(channel) == 1)), \
'error, channel must be None or a 1-letter string'
assert(bmnum == None or isinstance(bmnum,int)), \
'error, bmnum must be an int or None'
assert(cp == None or isinstance(cp,int)), \
'error, cp must be an int or None'
assert(fileType == 'rawacf' or fileType == 'fitacf' or \
fileType == 'fitex' or fileType == 'lmfit' or fileType == 'iqdat'), \
'error, fileType must be one of: rawacf,fitacf,fitex,lmfit,iqdat'
assert(fileName == None or isinstance(fileName,str)), \
'error, fileName must be None or a string'
assert(isinstance(filtered,bool)), \
'error, filtered must be True of False'
assert(src == None or src == 'mongo' or src == 'local' or src == 'sftp'), \
'error, src must be one of None,local,mongo,sftp'
if(eTime == None):
eTime = sTime+dt.timedelta(days=1)
#create a datapointer object
myPtr = radDataPtr(sTime=sTime,eTime=eTime,stid=int(network().getRadarByCode(rad).id),
channel=channel,bmnum=bmnum,cp=cp)
filelist = []
if(fileType == 'fitex'): arr = ['fitex','fitacf','lmfit']
elif(fileType == 'fitacf'): arr = ['fitacf','fitex','lmfit']
elif(fileType == 'lmfit'): arr = ['lmfit','fitex','fitacf']
else: arr = [fileType]
#move back a little in time because files often start at 2 mins after the hour
sTime = sTime-dt.timedelta(minutes=4)
#a temporary directory to store a temporary file
tmpDir = '/tmp/fit/'
d = os.path.dirname(tmpDir)
if not os.path.exists(d):
os.makedirs(d)
#FIRST, check if a specific filename was given
if(fileName != None):
try:
if(not os.path.isfile(fileName)):
print 'problem reading',fileName,':file does not exist'
return None
outname = tmpDir+str(int(datetimeToEpoch(dt.datetime.now())))
if(string.find(fileName,'.bz2') != -1):
outname = string.replace(fileName,'.bz2','')
print 'bunzip2 -c '+fileName+' > '+outname+'\n'
os.system('bunzip2 -c '+fileName+' > '+outname)
elif(string.find(fileName,'.gz') != -1):
outname = string.replace(fileName,'.gz','')
print 'gunzip -c '+fileName+' > '+outname+'\n'
os.system('gunzip -c '+fileName+' > '+outname)
else:
os.system('cp '+fileName+' '+outname)
print 'cp '+fileName+' '+outname
filelist.append(outname)
myPtr.fType,myPtr.dType = custType,'dmap'
except Exception, e:
print e
print 'problem reading file',fileName
return None
if(len(filelist) > 0):
print 'found',ftype,'data on sftp server'
myPtr.fType,myPtr.dType = ftype,'dmap'
break
if(len(filelist) > 0): break
else:
print 'could not find',ftype,'data on sftp server'
except Exception,e:
print e
print 'problem reading from sftp server'
#check if we have found files
if(len(filelist) != 0):
#concatenate the files into a single file
print 'Concatenating all the files in to one'
tmpName = tmpDir+str(int(datetimeToEpoch(dt.datetime.now())))+'.'+rad+'.'+fileType
print 'cat '+string.join(filelist)+' > '+tmpName
os.system('cat '+string.join(filelist)+' > '+tmpName)
for filename in filelist:
print 'rm '+filename
os.system('rm '+filename)
#filter(if desired) and open the file
if(not filtered): myPtr.ptr = open(tmpName,'r')
else:
print 'fitexfilter '+tmpName+' > '+tmpName+'f'
os.system('fitexfilter '+tmpName+' > '+tmpName+'f')
os.system('rm '+tmpName)
myPtr.ptr = open(tmpName+'f','r')
if(myPtr.ptr != None):
