def sdDataOpen(sTime,hemi='north',eTime=None,fileType='grdex',src=None,fileName=None, \ custType='grdex',noCache=False): """A function to establish a pipeline through which we can read radar data. first it tries the mongodb, then it tries to find local files, and lastly it sftp's over to the VT data server. **Args**: * **sTime** (`datetime <http://tinyurl.com/bl352yx>`_): the beginning time for which you want data * **[hemi]** (str): the hemisphere for which you want data, 'north' or 'south'. default = 'north' * **[eTime]** (`datetime <http://tinyurl.com/bl352yx>`_): the last time that you want data for. if this is set to None, it will be set to 1 day after sTime. default = None * **[fileType]** (str): The type of data you want to read. valid inputs are: 'grd','grdex','map','mapex'. If you choose a file format and the specified one isn't found, we will search for one of the others (eg mapex instead of map). default = 'grdex'. * **[src]** (str): the source of the data. valid inputs are 'local' 'sftp'. if this is set to None, it will try all possibilites sequentially. default = None * **[fileName]** (str): the name of a specific file which you want to open. If this is set, we will not look for cached files. default=None * **[custType]** (str): if fileName is specified, the filetype of the file. default = 'grdex' * **[noCache]** (boolean): flag to indicate that you do not want to check first for cached files. default = False. **Returns**: * **myPtr** (:class:`pydarn.sdio.sdDataTypes.sdDataPtr`): a sdDataPtr object which contains a link to the data to be read. this can then be passed to sdDataReadRec in order to actually read the data. **Example**: :: import datetime as dt myPtr = sdDataOpen(dt.datetime(2011,1,1),hemi='north'): Written by AJ 20130607 """ import paramiko as p import re import string import datetime as dt import os import pydarn.sdio import glob from pydarn.sdio import sdDataPtr from pydarn.radar import network from utils.timeUtils import datetimeToEpoch #check inputs assert(isinstance(sTime,dt.datetime)), \ 'error, sTime must be datetime object' assert(hemi == 'north' or hemi == 'south'), \ "error, hemi must be 'north' or 'south'" assert(eTime == None or isinstance(eTime,dt.datetime)), \ 'error, eTime must be datetime object or None' assert(fileType == 'grd' or fileType == 'grdex' or \ fileType == 'map' or fileType == 'mapex'), \ "error, fileType must be one of: 'grd','grdex','map','mapex'" assert(fileName == None or isinstance(fileName,str)), \ 'error, fileName must be None or a string' assert(src == None or src == 'local' or src == 'sftp'), \ 'error, src must be one of None,local,sftp' if eTime == None: eTime = sTime+dt.timedelta(days=1) #create a datapointer object myPtr = sdDataPtr(sTime=sTime,eTime=eTime,hemi=hemi) filelist = [] if fileType == 'grd': arr = ['grd','grdex'] elif fileType == 'grdex': arr = ['grdex','grd'] elif fileType == 'map': arr = ['map','mapex'] elif fileType == 'mapex': arr = ['mapex','map'] else: arr = [fileType] #move back a little in time because files often start at 2 mins after the hour sTime = sTime-dt.timedelta(minutes=4) #a temporary directory to store a temporary file tmpDir = '/tmp/sd/' d = os.path.dirname(tmpDir) if not os.path.exists(d): os.makedirs(d) cached = False fileSt = None #FIRST, check if a specific filename was given if fileName != None: try: if(not os.path.isfile(fileName)): print 'problem reading',fileName,':file does not exist' return None outname = tmpDir+str(int(datetimeToEpoch(dt.datetime.now()))) if(string.find(fileName,'.bz2') != -1): outname = string.replace(fileName,'.bz2','') print 'bunzip2 -c '+fileName+' > '+outname+'\n' os.system('bunzip2 -c '+fileName+' > '+outname) elif(string.find(fileName,'.gz') != -1): outname = string.replace(fileName,'.gz','') print 'gunzip -c '+fileName+' > '+outname+'\n' os.system('gunzip -c '+fileName+' > '+outname) else: os.system('cp '+fileName+' '+outname) print 'cp '+fileName+' '+outname filelist.append(outname) myPtr.fType,myPtr.dType = custType,'dmap' fileSt = sTime except Exception, e: print e print 'problem reading file',fileName return None
def radDataOpen(sTime,radcode,eTime=None,channel=None,bmnum=None,cp=None, \ fileType='fitex',filtered=False, src=None,fileName=None, \ custType='fitex',noCache=False): """A function to establish a pipeline through which we can read radar data. first it tries the mongodb, then it tries to find local files, and lastly it sftp's over to the VT data server. **Args**: * **sTime** (`datetime <http://tinyurl.com/bl352yx>`_): the beginning time for which you want data * **radcode** (str): the 3-letter radar code with optional channel extension for which you want data * **[eTime]** (`datetime <http://tinyurl.com/bl352yx>`_): the last time that you want data for. if this is set to None, it will be set to 1 day after sTime. default = None * **[channel]** (str): the 1-letter code for what channel you want data from, eg 'a','b',... if this is set to None, data from ALL channels will be read. default = None * **[bmnum]** (int): the beam number which you want data for. If this is set to None, data from all beams will be read. default = None * **[cp]** (int): the control program which you want data for. If this is set to None, data from all cp's will be read. default = None * **[fileType]** (str): The type of data you want to read. valid inputs are: 'fitex','fitacf','rawacf','iqdat'. if you choose a fit file format and the specified one isn't found, we will search for one of the others. Beware: if you ask for rawacf/iq data, these files are large and the data transfer might take a long time. default = 'fitex' * **[filtered]** (boolean): a boolean specifying whether you want the fit data to be boxcar filtered. ONLY VALID FOR FIT. default = False * **[src]** (str): the source of the data. valid inputs are 'local' 'sftp'. if this is set to None, it will try all possibilites sequentially. default = None * **[fileName]** (str): the name of a specific file which you want to open. default=None * **[custType]** (str): if fileName is specified, the filetype of the file. default='fitex' * **[noCache]** (boolean): flag to indicate that you do not want to check first for cached files. default = False. **Returns**: * **myPtr** (:class:`pydarn.sdio.radDataTypes.radDataPtr`): a radDataPtr object which contains a link to the data to be read. this can then be passed to radDataReadRec in order to actually read the data. **ENVIRONMENT Variables**: * DAVIT_TMPDIR : Directory used for davitpy temporary file cache. * DAVIT_TMPEXPIRE : Length of time that cached temporary files are valid. After which they will be regenerated. Example: DAVIT_TMPEXPIRE='2h' will reuse temp files in the cache for 2 hours since last access * DAVIT_LOCALDIR : Used to set base directory tree for local file look up * DAVIT_DIRFORMAT : Python string dictionary capable format string appended to local file base directory tree for use with directory structures which encode radar name, channel or date information. Currently supported dictionary keys which can be used: "dirtree" : base directory tree "year" : 0 padded 4 digit year "month" : 0 padded 2 digit month "day" : 0 padded 2 digit day "ftype" : filetype string "radar" : 3-chr radarcode **Example**: :: import datetime as dt myPtr = radDataOpen(dt.datetime(2011,1,1),'bks',eTime=dt.datetime(2011,1,1,2),channel='a', bmnum=7,cp=153,fileType='fitex',filtered=False, src=None): Written by AJ 20130110 """ import paramiko as p import re import string import datetime as dt, os, pydarn.sdio, glob from pydarn.sdio import radDataPtr from pydarn.radar import network from utils.timeUtils import datetimeToEpoch #check inputs assert(isinstance(sTime,dt.datetime)), \ 'error, sTime must be datetime object' segments=radcode.split(".") try: rad=segments[0] except: rad=None try: chan=segments[1] except: chan=None assert(isinstance(rad,str) and len(rad) == 3), \ 'error, rad must be a 3 char string' assert(eTime == None or isinstance(eTime,dt.datetime)), \ 'error, eTime must be datetime object or None' assert(channel == None or (isinstance(channel,str) and len(channel) == 1)), \ 'error, channel must be None or a 1-letter string' assert(bmnum == None or isinstance(bmnum,int)), \ 'error, bmnum must be an int or None' assert(cp == None or isinstance(cp,int)), \ 'error, cp must be an int or None' assert(fileType == 'rawacf' or fileType == 'fitacf' or \ fileType == 'fitex' or fileType == 'iqdat'), \ 'error, fileType must be one of: rawacf,fitacf,fitex,iqdat' assert(fileName == None or isinstance(fileName,str)), \ 'error, fileName must be None or a string' assert(isinstance(filtered,bool)), \ 'error, filtered must be True of False' assert(src == None or src == 'local' or src == 'sftp'), \ 'error, src must be one of None,local,sftp' if(eTime == None): eTime = sTime+dt.timedelta(days=1) #create a datapointer object myPtr = radDataPtr(sTime=sTime,eTime=eTime,stid=int(network().getRadarByCode(rad).id), channel=channel,bmnum=bmnum,cp=cp) filelist = [] # if(fileType == 'fitex'): arr = ['fitex','fitacf','lmfit'] # elif(fileType == 'fitacf'): arr = ['fitacf','fitex','lmfit'] # elif(fileType == 'lmfit'): arr = ['lmfit','fitex','fitacf'] if(fileType == 'fitex'): arr = ['fitex','fitacf'] elif(fileType == 'fitacf'): arr = ['fitacf','fitex'] else: arr = [fileType] #move back a little in time because files often start at 2 mins after the hour sTime = sTime-dt.timedelta(minutes=4) #a temporary directory to store a temporary file try: tmpDir=os.environ['DAVIT_TMPDIR'] except: tmpDir = '/tmp/sd/' d = os.path.dirname(tmpDir) if not os.path.exists(d): os.makedirs(d) cached = False fileSt = None #FIRST, check if a specific filename was given if fileName != None: try: if(not os.path.isfile(fileName)): print 'problem reading',fileName,':file does not exist' return None outname = tmpDir+str(int(datetimeToEpoch(dt.datetime.now()))) if(string.find(fileName,'.bz2') != -1): outname = string.replace(fileName,'.bz2','') print 'bunzip2 -c '+fileName+' > '+outname+'\n' os.system('bunzip2 -c '+fileName+' > '+outname) elif(string.find(fileName,'.gz') != -1): outname = string.replace(fileName,'.gz','') print 'gunzip -c '+fileName+' > '+outname+'\n' os.system('gunzip -c '+fileName+' > '+outname) else: os.system('cp '+fileName+' '+outname) print 'cp '+fileName+' '+outname filelist.append(outname) myPtr.fType,myPtr.dType = custType,'dmap' fileSt = sTime except Exception, e: print e print 'problem reading file',fileName return None
def radDataOpen(sTime,radcode,eTime=None,channel=None,bmnum=None,cp=None, \ fileType='fitex',filtered=False, src=None,fileName=None, \ custType='fitex',noCache=False): """A function to establish a pipeline through which we can read radar data. first it tries the mongodb, then it tries to find local files, and lastly it sftp's over to the VT data server. **Args**: * **sTime** (`datetime <http://tinyurl.com/bl352yx>`_): the beginning time for which you want data * **radcode** (str): the 3-letter radar code with optional channel extension for which you want data * **[eTime]** (`datetime <http://tinyurl.com/bl352yx>`_): the last time that you want data for. if this is set to None, it will be set to 1 day after sTime. default = None * **[channel]** (str): the 1-letter code for what channel you want data from, eg 'a','b',... if this is set to None, data from ALL channels will be read. default = None * **[bmnum]** (int): the beam number which you want data for. If this is set to None, data from all beams will be read. default = None * **[cp]** (int): the control program which you want data for. If this is set to None, data from all cp's will be read. default = None * **[fileType]** (str): The type of data you want to read. valid inputs are: 'fitex','fitacf','rawacf','iqdat'. if you choose a fit file format and the specified one isn't found, we will search for one of the others. Beware: if you ask for rawacf/iq data, these files are large and the data transfer might take a long time. default = 'fitex' * **[filtered]** (boolean): a boolean specifying whether you want the fit data to be boxcar filtered. ONLY VALID FOR FIT. default = False * **[src]** (str): the source of the data. valid inputs are 'local' 'sftp'. if this is set to None, it will try all possibilites sequentially. default = None * **[fileName]** (str): the name of a specific file which you want to open. default=None * **[custType]** (str): if fileName is specified, the filetype of the file. default='fitex' * **[noCache]** (boolean): flag to indicate that you do not want to check first for cached files. default = False. **Returns**: * **myPtr** (:class:`pydarn.sdio.radDataTypes.radDataPtr`): a radDataPtr object which contains a link to the data to be read. this can then be passed to radDataReadRec in order to actually read the data. **ENVIRONMENT Variables**: * DAVIT_TMPDIR : Directory used for davitpy temporary file cache. * DAVIT_TMPEXPIRE : Length of time that cached temporary files are valid. After which they will be regenerated. Example: DAVIT_TMPEXPIRE='2h' will reuse temp files in the cache for 2 hours since last access * DAVIT_LOCALDIR : Used to set base directory tree for local file look up * DAVIT_DIRFORMAT : Python string dictionary capable format string appended to local file base directory tree for use with directory structures which encode radar name, channel or date information. Currently supported dictionary keys which can be used: "dirtree" : base directory tree "year" : 0 padded 4 digit year "month" : 0 padded 2 digit month "day" : 0 padded 2 digit day "ftype" : filetype string "radar" : 3-chr radarcode **Example**: :: import datetime as dt myPtr = radDataOpen(dt.datetime(2011,1,1),'bks',eTime=dt.datetime(2011,1,1,2),channel='a', bmnum=7,cp=153,fileType='fitex',filtered=False, src=None): Written by AJ 20130110 """ import paramiko as p import re import string import datetime as dt, os, pydarn.sdio, glob from pydarn.sdio import radDataPtr from pydarn.radar import network from utils.timeUtils import datetimeToEpoch #check inputs assert(isinstance(sTime,dt.datetime)), \ 'error, sTime must be datetime object' segments = radcode.split(".") try: rad = segments[0] except: rad = None try: chan = segments[1] except: chan = None assert(isinstance(rad,str) and len(rad) == 3), \ 'error, rad must be a 3 char string' assert(eTime == None or isinstance(eTime,dt.datetime)), \ 'error, eTime must be datetime object or None' assert(channel == None or (isinstance(channel,str) and len(channel) == 1)), \ 'error, channel must be None or a 1-letter string' assert(bmnum == None or isinstance(bmnum,int)), \ 'error, bmnum must be an int or None' assert(cp == None or isinstance(cp,int)), \ 'error, cp must be an int or None' assert(fileType == 'rawacf' or fileType == 'fitacf' or \ fileType == 'fitex' or fileType == 'iqdat'), \ 'error, fileType must be one of: rawacf,fitacf,fitex,iqdat' assert(fileName == None or isinstance(fileName,str)), \ 'error, fileName must be None or a string' assert(isinstance(filtered,bool)), \ 'error, filtered must be True of False' assert(src == None or src == 'local' or src == 'sftp'), \ 'error, src must be one of None,local,sftp' if (eTime == None): eTime = sTime + dt.timedelta(days=1) #create a datapointer object myPtr = radDataPtr(sTime=sTime, eTime=eTime, stid=int(network().getRadarByCode(rad).id), channel=channel, bmnum=bmnum, cp=cp) filelist = [] # if(fileType == 'fitex'): arr = ['fitex','fitacf','lmfit'] # elif(fileType == 'fitacf'): arr = ['fitacf','fitex','lmfit'] # elif(fileType == 'lmfit'): arr = ['lmfit','fitex','fitacf'] if (fileType == 'fitex'): arr = ['fitex', 'fitacf'] elif (fileType == 'fitacf'): arr = ['fitacf', 'fitex'] else: arr = [fileType] #move back a little in time because files often start at 2 mins after the hour sTime = sTime - dt.timedelta(minutes=4) #a temporary directory to store a temporary file try: tmpDir = os.environ['DAVIT_TMPDIR'] except: tmpDir = '/tmp/sd/' d = os.path.dirname(tmpDir) if not os.path.exists(d): os.makedirs(d) cached = False fileSt = None #FIRST, check if a specific filename was given if fileName != None: try: if (not os.path.isfile(fileName)): print 'problem reading', fileName, ':file does not exist' return None outname = tmpDir + str(int(datetimeToEpoch(dt.datetime.now()))) if (string.find(fileName, '.bz2') != -1): outname = string.replace(fileName, '.bz2', '') print 'bunzip2 -c ' + fileName + ' > ' + outname + '\n' os.system('bunzip2 -c ' + fileName + ' > ' + outname) elif (string.find(fileName, '.gz') != -1): outname = string.replace(fileName, '.gz', '') print 'gunzip -c ' + fileName + ' > ' + outname + '\n' os.system('gunzip -c ' + fileName + ' > ' + outname) else: os.system('cp ' + fileName + ' ' + outname) print 'cp ' + fileName + ' ' + outname filelist.append(outname) myPtr.fType, myPtr.dType = custType, 'dmap' fileSt = sTime except Exception, e: print e print 'problem reading file', fileName return None
def sdDataOpen(sTime,hemi='north',eTime=None,fileType='grdex',src=None,fileName=None, \ custType='grdex',noCache=False): """A function to establish a pipeline through which we can read radar data. first it tries the mongodb, then it tries to find local files, and lastly it sftp's over to the VT data server. **Args**: * **sTime** (`datetime <http://tinyurl.com/bl352yx>`_): the beginning time for which you want data * **[hemi]** (str): the hemisphere for which you want data, 'north' or 'south'. default = 'north' * **[eTime]** (`datetime <http://tinyurl.com/bl352yx>`_): the last time that you want data for. if this is set to None, it will be set to 1 day after sTime. default = None * **[fileType]** (str): The type of data you want to read. valid inputs are: 'grd','grdex','map','mapex'. If you choose a file format and the specified one isn't found, we will search for one of the others (eg mapex instead of map). default = 'grdex'. * **[src]** (str): the source of the data. valid inputs are 'local' 'sftp'. if this is set to None, it will try all possibilites sequentially. default = None * **[fileName]** (str): the name of a specific file which you want to open. If this is set, we will not look for cached files. default=None * **[custType]** (str): if fileName is specified, the filetype of the file. default = 'grdex' * **[noCache]** (boolean): flag to indicate that you do not want to check first for cached files. default = False. **Returns**: * **myPtr** (:class:`pydarn.sdio.sdDataTypes.sdDataPtr`): a sdDataPtr object which contains a link to the data to be read. this can then be passed to sdDataReadRec in order to actually read the data. **Example**: :: import datetime as dt myPtr = sdDataOpen(dt.datetime(2011,1,1),hemi='north'): Written by AJ 20130607 """ import paramiko as p import re import string import datetime as dt import os import pydarn.sdio import glob from pydarn.sdio import sdDataPtr from pydarn.radar import network from utils.timeUtils import datetimeToEpoch #check inputs assert(isinstance(sTime,dt.datetime)), \ 'error, sTime must be datetime object' assert(hemi == 'north' or hemi == 'south'), \ "error, hemi must be 'north' or 'south'" assert(eTime == None or isinstance(eTime,dt.datetime)), \ 'error, eTime must be datetime object or None' assert(fileType == 'grd' or fileType == 'grdex' or \ fileType == 'map' or fileType == 'mapex'), \ "error, fileType must be one of: 'grd','grdex','map','mapex'" assert(fileName == None or isinstance(fileName,str)), \ 'error, fileName must be None or a string' assert(src == None or src == 'local' or src == 'sftp'), \ 'error, src must be one of None,local,sftp' if eTime == None: eTime = sTime + dt.timedelta(days=1) #create a datapointer object myPtr = sdDataPtr(sTime=sTime, eTime=eTime, hemi=hemi) filelist = [] if fileType == 'grd': arr = ['grd', 'grdex'] elif fileType == 'grdex': arr = ['grdex', 'grd'] elif fileType == 'map': arr = ['map', 'mapex'] elif fileType == 'mapex': arr = ['mapex', 'map'] else: arr = [fileType] #move back a little in time because files often start at 2 mins after the hour sTime = sTime - dt.timedelta(minutes=4) #a temporary directory to store a temporary file tmpDir = '/tmp/sd/' d = os.path.dirname(tmpDir) if not os.path.exists(d): os.makedirs(d) cached = False fileSt = None #FIRST, check if a specific filename was given if fileName != None: try: if (not os.path.isfile(fileName)): print 'problem reading', fileName, ':file does not exist' return None outname = tmpDir + str(int(datetimeToEpoch(dt.datetime.now()))) if (string.find(fileName, '.bz2') != -1): outname = string.replace(fileName, '.bz2', '') print 'bunzip2 -c ' + fileName + ' > ' + outname + '\n' os.system('bunzip2 -c ' + fileName + ' > ' + outname) elif (string.find(fileName, '.gz') != -1): outname = string.replace(fileName, '.gz', '') print 'gunzip -c ' + fileName + ' > ' + outname + '\n' os.system('gunzip -c ' + fileName + ' > ' + outname) else: os.system('cp ' + fileName + ' ' + outname) print 'cp ' + fileName + ' ' + outname filelist.append(outname) myPtr.fType, myPtr.dType = custType, 'dmap' fileSt = sTime except Exception, e: print e print 'problem reading file', fileName return None
def radDataOpen(sTime,rad,eTime=None,channel=None,bmnum=None,cp=None, \ fileType='fitex',filtered=False, src=None,fileName=None, \ custType='fitex'): """A function to establish a pipeline through which we can read radar data. first it tries the mongodb, then it tries to find local files, and lastly it sftp's over to the VT data server. **Args**: * **sTime** (`datetime <http://tinyurl.com/bl352yx>`_): the beginning time for which you want data * **rad** (str): the 3-letter radar code for which you want data * **[eTime]** (`datetime <http://tinyurl.com/bl352yx>`_): the last time that you want data for. if this is set to None, it will be set to 1 day after sTime. default = None * **[channel]** (str): the 1-letter code for what channel you want data from, eg 'a','b',... if this is set to None, data from ALL channels will be read. default = None * **[bmnum]** (int): the beam number which you want data for. If this is set to None, data from all beams will be read. default = None * **[cp]** (int): the control program which you want data for. If this is set to None, data from all cp's will be read. default = None * **[fileType]** (str): The type of data you want to read. valid inputs are: 'fitex','fitacf','lmfit','rawacf','iqdat'. if you choose a fit file format and the specified one isn't found, we will search for one of the others. Beware: if you ask for rawacf/iq data, these files are large and the data transfer might take a long time. default = 'fitex' * **[filtered]** (boolean): a boolean specifying whether you want the fit data to be boxcar filtered. ONLY VALID FOR FIT. default = False * **[src]** (str): the source of the data. valid inputs are 'mongo' 'local' 'sftp'. if this is set to None, it will try all possibilites sequentially. default = None * **[fileName]** (str): the name of a specific file which you want to open. default=None * **[custType]** (str): if fileName is specified, the filetype of the file. default='fitex' **Returns**: * **myPtr** (:class:`radDataTypes.radDataPtr`): a radDataPtr object which contains a link to the data to be read. this can then be passed to radDataReadRec in order to actually read the data. **Example**: :: import datetime as dt myPtr = radDataOpen(dt.datetime(2011,1,1),'bks',eTime=dt.datetime(2011,1,1,2),channel='a', bmnum=7,cp=153,fileType='fitex',filtered=False, src=None): Written by AJ 20130110 """ import subprocess as sub, paramiko as p, re, string import datetime as dt, os, pydarn.sdio, glob from pydarn.sdio import radDataPtr from pydarn.radar import network from utils.timeUtils import datetimeToEpoch #check inputs assert(isinstance(sTime,dt.datetime)), \ 'error, sTime must be datetime object' assert(isinstance(rad,str) and len(rad) == 3), \ 'error, rad must be a 3 char string' assert(eTime == None or isinstance(eTime,dt.datetime)), \ 'error, eTime must be datetime object or None' assert(channel == None or (isinstance(channel,str) and len(channel) == 1)), \ 'error, channel must be None or a 1-letter string' assert(bmnum == None or isinstance(bmnum,int)), \ 'error, bmnum must be an int or None' assert(cp == None or isinstance(cp,int)), \ 'error, cp must be an int or None' assert(fileType == 'rawacf' or fileType == 'fitacf' or \ fileType == 'fitex' or fileType == 'lmfit' or fileType == 'iqdat'), \ 'error, fileType must be one of: rawacf,fitacf,fitex,lmfit,iqdat' assert(fileName == None or isinstance(fileName,str)), \ 'error, fileName must be None or a string' assert(isinstance(filtered,bool)), \ 'error, filtered must be True of False' assert(src == None or src == 'mongo' or src == 'local' or src == 'sftp'), \ 'error, src must be one of None,local,mongo,sftp' if(eTime == None): eTime = sTime+dt.timedelta(days=1) #create a datapointer object myPtr = radDataPtr(sTime=sTime,eTime=eTime,stid=int(network().getRadarByCode(rad).id), channel=channel,bmnum=bmnum,cp=cp) filelist = [] if(fileType == 'fitex'): arr = ['fitex','fitacf','lmfit'] elif(fileType == 'fitacf'): arr = ['fitacf','fitex','lmfit'] elif(fileType == 'lmfit'): arr = ['lmfit','fitex','fitacf'] else: arr = [fileType] #move back a little in time because files often start at 2 mins after the hour sTime = sTime-dt.timedelta(minutes=4) #a temporary directory to store a temporary file tmpDir = '/tmp/fit/' d = os.path.dirname(tmpDir) if not os.path.exists(d): os.makedirs(d) #FIRST, check if a specific filename was given if(fileName != None): try: if(not os.path.isfile(fileName)): print 'problem reading',fileName,':file does not exist' return None outname = tmpDir+str(int(datetimeToEpoch(dt.datetime.now()))) if(string.find(fileName,'.bz2') != -1): outname = string.replace(fileName,'.bz2','') print 'bunzip2 -c '+fileName+' > '+outname+'\n' os.system('bunzip2 -c '+fileName+' > '+outname) elif(string.find(fileName,'.gz') != -1): outname = string.replace(fileName,'.gz','') print 'gunzip -c '+fileName+' > '+outname+'\n' os.system('gunzip -c '+fileName+' > '+outname) else: os.system('cp '+fileName+' '+outname) print 'cp '+fileName+' '+outname filelist.append(outname) myPtr.fType,myPtr.dType = custType,'dmap' except Exception, e: print e print 'problem reading file',fileName return None
if(len(filelist) > 0): print 'found',ftype,'data on sftp server' myPtr.fType,myPtr.dType = ftype,'dmap' break if(len(filelist) > 0): break else: print 'could not find',ftype,'data on sftp server' except Exception,e: print e print 'problem reading from sftp server' #check if we have found files if(len(filelist) != 0): #concatenate the files into a single file print 'Concatenating all the files in to one' tmpName = tmpDir+str(int(datetimeToEpoch(dt.datetime.now())))+'.'+rad+'.'+fileType print 'cat '+string.join(filelist)+' > '+tmpName os.system('cat '+string.join(filelist)+' > '+tmpName) for filename in filelist: print 'rm '+filename os.system('rm '+filename) #filter(if desired) and open the file if(not filtered): myPtr.ptr = open(tmpName,'r') else: print 'fitexfilter '+tmpName+' > '+tmpName+'f' os.system('fitexfilter '+tmpName+' > '+tmpName+'f') os.system('rm '+tmpName) myPtr.ptr = open(tmpName+'f','r') if(myPtr.ptr != None):