def mapDstMongo(sYear, eYear=None): """This function reads dst data from wdc and puts it in mongodb .. warning:: In general, nobody except the database admins will need to use this function **Args**: * **sYear** (int): the year to begin mapping data * [**eYear**] (int or None): the end year for mapping data. if this is None, eYear will be sYear **Returns**: * Nothing. **Example**: :: gme.ind.mapDstMongo(1997) written by AJ, 20130123 """ import davitpy.pydarn.sdio.dbUtils as db from davitpy import rcParams import datetime as dt #check inputs assert (isinstance(sYear, int)), 'error, sYear must be int' if (eYear == None): eYear = sYear assert (isinstance(eYear, int)), 'error, sYear must be None or int' assert (eYear >= sYear), 'error, end year greater than start year' #get data connection mongoData = db.getDataConn(username=rcParams['DBWRITEUSER'],password=rcParams['DBWRITEPASS'],\ dbAddress=rcParams['SDDB'],dbName='gme',collName='dst') #set up all of the indices mongoData.ensure_index('time') mongoData.ensure_index('dst') for yr in range(sYear, eYear + 1): #1 year at a time, to not fill up RAM templist = readDstWeb(dt.datetime(yr, 1, 1), dt.datetime(yr, 12, 31)) for rec in templist: #check if a duplicate record exists qry = mongoData.find({'time': rec.time}) print rec.time tempRec = rec.toDbDict() cnt = qry.count() #if this is a new record, insert it if (cnt == 0): mongoData.insert(tempRec) #if this is an existing record, update it elif (cnt == 1): print 'foundone!!' dbDict = qry.next() temp = dbDict['_id'] dbDict = tempRec dbDict['_id'] = temp mongoData.save(dbDict) else: print 'strange, there is more than 1 DST record for', rec.time del templist
def mapKpMongo(sYear, eYear=None): """This function reads kp data from the GFZ Potsdam FTP server via anonymous FTP connection and maps it to the mongodb. .. warning:: In general, nobody except the database admins will need to use this function **Args**: * **sYear** (int): the year to begin mapping data * [**eYear**] (int or None): the end year for mapping data. if this is None, eYear will be sYear. default=None **Returns**: * Nothing. **Example**: :: gme.ind.mapKpMongo(1985,eTime=1986) written by AJ, 20130123 """ import davitpy.pydarn.sdio.dbUtils as db from davitpy import rcParams import datetime as dt if (eYear == None): eYear = sYear assert (eYear >= sYear), 'error, end year greater than start year' mongoData = db.getDataConn(username=rcParams['DBWRITEUSER'],password=rcParams['DBWRITEPASS'],\ dbAddress=rcParams['SDDB'],dbName='gme',collName='kp') #set up all of the indices mongoData.ensure_index('time') mongoData.ensure_index('kp') mongoData.ensure_index('ap') mongoData.ensure_index('kpSum') mongoData.ensure_index('apMean') mongoData.ensure_index('sunspot') #read the kp data from the FTP server datalist = [] for yr in range(sYear, eYear + 1): templist = readKpFtp(dt.datetime(yr, 1, 1), dt.datetime(yr + 1, 1, 1)) if (templist == None): continue for rec in templist: #check if a duplicate record exists qry = mongoData.find({'time': rec.time}) tempRec = rec.toDbDict() cnt = qry.count() #if this is a new record, insert it if (cnt == 0): mongoData.insert(tempRec) #if this is an existing record, update it elif (cnt == 1): print 'foundone!!' dbDict = qry.next() temp = dbDict['_id'] dbDict = tempRec dbDict['_id'] = temp mongoData.save(dbDict) else: print 'strange, there is more than 1 record for', rec.time
def mapDstMongo(sYear,eYear=None): """This function reads dst data from wdc and puts it in mongodb .. warning:: In general, nobody except the database admins will need to use this function **Args**: * **sYear** (int): the year to begin mapping data * [**eYear**] (int or None): the end year for mapping data. if this is None, eYear will be sYear **Returns**: * Nothing. **Example**: :: gme.ind.mapDstMongo(1997) written by AJ, 20130123 """ import davitpy.pydarn.sdio.dbUtils as db from davitpy import rcParams import datetime as dt #check inputs assert(isinstance(sYear,int)),'error, sYear must be int' if(eYear == None): eYear=sYear assert(isinstance(eYear,int)),'error, sYear must be None or int' assert(eYear >= sYear), 'error, end year greater than start year' #get data connection mongoData = db.getDataConn(username=rcParams['DBWRITEUSER'],password=rcParams['DBWRITEPASS'],\ dbAddress=rcParams['SDDB'],dbName='gme',collName='dst') #set up all of the indices mongoData.ensure_index('time') mongoData.ensure_index('dst') for yr in range(sYear,eYear+1): #1 year at a time, to not fill up RAM templist = readDstWeb(dt.datetime(yr,1,1),dt.datetime(yr,12,31)) for rec in templist: #check if a duplicate record exists qry = mongoData.find({'time':rec.time}) print rec.time tempRec = rec.toDbDict() cnt = qry.count() #if this is a new record, insert it if(cnt == 0): mongoData.insert(tempRec) #if this is an existing record, update it elif(cnt == 1): print 'foundone!!' dbDict = qry.next() temp = dbDict['_id'] dbDict = tempRec dbDict['_id'] = temp mongoData.save(dbDict) else: print 'strange, there is more than 1 DST record for',rec.time del templist
def mapKpMongo(sYear,eYear=None): """This function reads kp data from the GFZ Potsdam FTP server via anonymous FTP connection and maps it to the mongodb. .. warning:: In general, nobody except the database admins will need to use this function **Args**: * **sYear** (int): the year to begin mapping data * [**eYear**] (int or None): the end year for mapping data. if this is None, eYear will be sYear. default=None **Returns**: * Nothing. **Example**: :: gme.ind.mapKpMongo(1985,eTime=1986) written by AJ, 20130123 """ import davitpy.pydarn.sdio.dbUtils as db from davitpy import rcParams import datetime as dt if(eYear == None): eYear=sYear assert(eYear >= sYear), 'error, end year greater than start year' mongoData = db.getDataConn(username=rcParams['DBWRITEUSER'],password=rcParams['DBWRITEPASS'],\ dbAddress=rcParams['SDDB'],dbName='gme',collName='kp') #set up all of the indices mongoData.ensure_index('time') mongoData.ensure_index('kp') mongoData.ensure_index('ap') mongoData.ensure_index('kpSum') mongoData.ensure_index('apMean') mongoData.ensure_index('sunspot') #read the kp data from the FTP server datalist = [] for yr in range(sYear,eYear+1): templist = readKpFtp(dt.datetime(yr,1,1), dt.datetime(yr+1,1,1)) if(templist == None): continue for rec in templist: #check if a duplicate record exists qry = mongoData.find({'time':rec.time}) tempRec = rec.toDbDict() cnt = qry.count() #if this is a new record, insert it if(cnt == 0): mongoData.insert(tempRec) #if this is an existing record, update it elif(cnt == 1): print 'foundone!!' dbDict = qry.next() temp = dbDict['_id'] dbDict = tempRec dbDict['_id'] = temp mongoData.save(dbDict) else: print 'strange, there is more than 1 record for',rec.time
def readDst(sTime=None,eTime=None,dst=None): """This function reads dst data from the mongodb. **Args**: * [**sTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the earliest time you want data for, default=None * [**eTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the latest time you want data for. if this is None, end Time will be 1 day after sTime. default = None * [**dst**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with dst values in the range [a,b] will be returned. default = None **Returns**: * **dstList** (list or None): if data is found, a list of :class:`gme.ind.dst.dstRec` objects matching the input parameters is returned. If no data is found, None is returned. **Example**: :: import datetime as dt dstList = gme.ind.readDst(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),dst=[-50,50]) written by AJ, 20130131 """ import datetime as dt import davitpy.pydarn.sdio.dbUtils as db #check all the inputs for validity assert(sTime == None or isinstance(sTime,dt.datetime)), \ 'error, sTime must be a datetime object' assert(eTime == None or isinstance(eTime,dt.datetime)), \ 'error, eTime must be either None or a datetime object' assert(dst == None or (isinstance(dst,list) and \ isinstance(dst[0],(int,float)) and isinstance(dst[1],(int,float)))), \ 'error,dst must None or a list of 2 numbers' if(eTime == None and sTime != None): eTime = sTime+dt.timedelta(days=1) qryList = [] #if arguments are provided, query for those if(sTime != None): qryList.append({'time':{'$gte':sTime}}) if(eTime != None): qryList.append({'time':{'$lte':eTime}}) if(dst != None): qryList.append({'dst':{'$gte':min(dst)}}) qryList.append({'dst':{'$lte':max(dst)}}) #construct the final query definition qryDict = {'$and': qryList} #connect to the database dstData = db.getDataConn(dbName='gme',collName='dst') #do the query if(qryList != []): qry = dstData.find(qryDict) else: qry = dstData.find() if(qry.count() > 0): dstList = [] for rec in qry.sort('time'): dstList.append(dstRec(dbDict=rec)) print '\nreturning a list with',len(dstList),'records of dst data' return dstList #if we didn't find anything on the mongodb else: print '\ncould not find requested data in the mongodb' return None
def readDst(sTime=None, eTime=None, dst=None): """This function reads dst data from the mongodb. **Args**: * [**sTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the earliest time you want data for, default=None * [**eTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the latest time you want data for. if this is None, end Time will be 1 day after sTime. default = None * [**dst**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with dst values in the range [a,b] will be returned. default = None **Returns**: * **dstList** (list or None): if data is found, a list of :class:`gme.ind.dst.dstRec` objects matching the input parameters is returned. If no data is found, None is returned. **Example**: :: import datetime as dt dstList = gme.ind.readDst(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),dst=[-50,50]) written by AJ, 20130131 """ import datetime as dt import davitpy.pydarn.sdio.dbUtils as db #check all the inputs for validity assert(sTime == None or isinstance(sTime,dt.datetime)), \ 'error, sTime must be a datetime object' assert(eTime == None or isinstance(eTime,dt.datetime)), \ 'error, eTime must be either None or a datetime object' assert(dst == None or (isinstance(dst,list) and \ isinstance(dst[0],(int,float)) and isinstance(dst[1],(int,float)))), \ 'error,dst must None or a list of 2 numbers' if (eTime == None and sTime != None): eTime = sTime + dt.timedelta(days=1) qryList = [] #if arguments are provided, query for those if (sTime != None): qryList.append({'time': {'$gte': sTime}}) if (eTime != None): qryList.append({'time': {'$lte': eTime}}) if (dst != None): qryList.append({'dst': {'$gte': min(dst)}}) qryList.append({'dst': {'$lte': max(dst)}}) #construct the final query definition qryDict = {'$and': qryList} #connect to the database dstData = db.getDataConn(dbName='gme', collName='dst') #do the query if (qryList != []): qry = dstData.find(qryDict) else: qry = dstData.find() if (qry.count() > 0): dstList = [] for rec in qry.sort('time'): dstList.append(dstRec(dbDict=rec)) print '\nreturning a list with', len(dstList), 'records of dst data' return dstList #if we didn't find anything on the mongodb else: print '\ncould not find requested data in the mongodb' return None
def mapOmniMongo(sYear,eYear=None,res=5): """This function reads omni data from the NASA SPDF FTP server via anonymous FTP connection and maps it to the mongodb. .. warning:: In general, nobody except the database admins will need to use this function **Args**: * **sYear** (int): the year to begin mapping data * [**eYear**] (int or None): the end year for mapping data. if this is None, eYear will be sYear * [**res**] (int): the time resolution for mapping data. Can be either 1 or 5. default=5 **Returns**: * Nothing. **Example**: :: gme.ind.mapOmniMongo(1997,res=1) written by AJ, 20130123 """ import davitpy.pydarn.sdio.dbUtils as db from davitpy import rcParams import datetime as dt #check inputs assert(res == 1 or res == 5),'error, res must be either 1 or 5' assert(isinstance(sYear,int)),'error, sYear must be int' if(eYear == None): eYear=sYear assert(isinstance(eYear,int)),'error, sYear must be None or int' assert(eYear >= sYear), 'error, end year greater than start year' #get data connection mongoData = db.getDataConn(username=rcParams['DBWRITEUSER'],password=rcParams['DBWRITEPASS'],\ dbAddress=rcParams['SDDB'],dbName='gme',collName='omni') #set up all of the indices mongoData.ensure_index('time') mongoData.ensure_index('res') mongoData.ensure_index('bx') mongoData.ensure_index('bye') mongoData.ensure_index('bze') mongoData.ensure_index('bym') mongoData.ensure_index('bzm') mongoData.ensure_index('pDyn') mongoData.ensure_index('ae') mongoData.ensure_index('symh') #read the omni data from the FTP server for yr in range(sYear,eYear+1): for mon in range(1,13): templist = readOmniFtp(dt.datetime(yr,mon,1),dt.datetime(yr,mon,1)+dt.timedelta(days=31),res=res) if(templist == None): continue for rec in templist: #check if a duplicate record exists qry = mongoData.find({'$and':[{'time':rec.time},{'res':rec.res}]}) print rec.time tempRec = rec.toDbDict() cnt = qry.count() #if this is a new record, insert it if(cnt == 0): mongoData.insert(tempRec) #if this is an existing record, update it elif(cnt == 1): print 'foundone!!' dbDict = qry.next() temp = dbDict['_id'] dbDict = tempRec dbDict['_id'] = temp mongoData.save(dbDict) else: print 'strange, there is more than 1 record for',rec.time
def readOmni(sTime,eTime=None,res=5,bx=None,bye=None,bze=None,bym=None,bzm=None,pDyn=None,ae=None,symh=None): """This function reads omni data. First, it will try to get it from the mongodb, and if it can't find it, it will look on the NASA SPDF FTP server using :func:`readOmniFtp` **Args**: * **sTime** (`datetime <http://tinyurl.com/bl352yx>`_ or None): the earliest time you want data for * [**eTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the latest time you want data for. if this is None, end Time will be 1 day after sTime. default = None * [**res**] (int): the time reolution of data desired. This can be either 1 or 5. default = 5 * [**bx**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bx values in the range [a,b] will be returned. default = None * [**bx**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bx values in the range [a,b] will be returned. default = None * [**bye**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bye values in the range [a,b] will be returned. default = None * [**bze**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bze values in the range [a,b] will be returned. default = None * [**bym**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bym values in the range [a,b] will be returned. default = None * [**bzm**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bzm values in the range [a,b] will be returned. default = None * [**pDyn**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with pDyn values in the range [a,b] will be returned. default = None * [**ae**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with ae values in the range [a,b] will be returned. default = None * [**symh**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with symh values in the range [a,b] will be returned. default = None **Returns**: * **omniList** (list or None): if data is found, a list of :class:`omniRec` objects matching the input parameters is returned. If no data is found, None is returned. **Example**: :: import datetime as dt omniList = gme.ind.readOmni(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),bx=[0,5.5],bye=[-1,3.5],bze=[-10,0],ae=[0,56.3]) written by AJ, 20130128 """ import datetime as dt import davitpy.pydarn.sdio.dbUtils as db #check all the inputs for validity assert(isinstance(sTime,dt.datetime)), \ 'error, sTime must be a datetime object' assert(eTime == None or isinstance(eTime,dt.datetime)), \ 'error, eTime must be either None or a datetime object' assert(res==5 or res == 1), 'error, res must be either 1 or 5' var = locals() for name in ['bx','bye','bze','bym','bzm','pDyn','ae','symh']: assert(var[name] == None or (isinstance(var[name],list) and \ isinstance(var[name][0],(int,float)) and isinstance(var[name][1],(int,float)))), \ 'error,'+name+' must None or a list of 2 numbers' if(eTime == None): eTime = sTime+dt.timedelta(days=1) qryList = [] #if arguments are provided, query for those qryList.append({'time':{'$gte':sTime}}) qryList.append({'res':res}) if(eTime != None): qryList.append({'time':{'$lte':eTime}}) var = locals() for name in ['bx','bye','bze','bym','bzm','pDyn','ae','symh']: if(var[name] != None): qryList.append({name:{'$gte':min(var[name])}}) qryList.append({name:{'$lte':max(var[name])}}) #construct the final query definition qryDict = {'$and': qryList} #connect to the database omniData = db.getDataConn(dbName='gme',collName='omni') #do the query if(qryList != []): qry = omniData.find(qryDict) else: qry = omniData.find() if(qry.count() > 0): omniList = [] for rec in qry.sort('time'): omniList.append(omniRec(dbDict=rec)) print '\nreturning a list with',len(omniList),'records of omni data' return omniList #if we didn't find anything on the mongodb else: print '\ncould not find requested data in the mongodb' print 'we will look on the ftp server, but your conditions will be (mostly) ignored' #read from ftp server omniList = readOmniFtp(sTime, eTime) if(omniList != None): print '\nreturning a list with',len(omniList),'recs of omni data' return omniList else: print '\n no data found on FTP server, returning None...' return None
def readPoes(stime, eTime=None, satnum=None, folat=None, folon=None, ted=None, echar=None, pchar=None): """This function reads POES data. Parameters ---------- stime : (datetime or None) the earliest time you want data for eTime : (datetime or None) the latest time you want data for. if this is None, end Time will be 1 day after stime. (default=None) satnum : (int) the satellite you want data for. eg 17 for noaa17. if this is None, data for all satellites will be returned. (default=None) folat : (list or None) if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bx values in the range [a,b] will be returned. (default=None) folon : (list or None) if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bye values in the range [a,b] will be returned (default=None) ted : (list or None) if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bze values in the range [a,b] will be returned (default=None) echar : (list or None) if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bym values in the range [a,b] will be returned (default=None) pchar : (list or None) if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bzm values in the range [a,b] will be returned (default=None) Returns ------- poesList : (list or None) if data is found, a list of class poesRec objects matching the input parameters is returned. If no data is found, None is returned. Notes -------- First, it will try to get it from the mongodb, and if it can't find it, it will look on the NOAA NGDC FTP server using :func:`readPoesFtp`. The data are 16-second averages Example ------- import datetime as dt poesList = gme.sat.readPoes(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),folat=[60,80]) written by AJ, 20130131 """ import datetime as dt import davitpy.pydarn.sdio.dbUtils as db # check all the inputs for validity assert isinstance(stime, dt.datetime), \ logging.error('stime must be a datetime object') assert eTime == None or isinstance(eTime,dt.datetime), \ logging.error('eTime must be either None or a datetime object') assert satnum == None or isinstance(satnum,int), \ logging.error('satnum must be an int') var = locals() for name in ['folat','folon','ted','echar','pchar']: assert(var[name] == None or (isinstance(var[name],list) and isinstance(var[name][0],(int,float)) and isinstance(var[name][1],(int,float)))), \ logging.error(name + ' must None or a list of 2 numbers') if(eTime == None): eTime = stime + dt.timedelta(days=1) qryList = [] # if arguments are provided, query for those qryList.append({'time':{'$gte':stime}}) if(eTime != None): qryList.append({'time':{'$lte':eTime}}) if(satnum != None): qryList.append({'satnum':satnum}) var = locals() for name in ['folat','folon','ted','echar','pchar']: if(var[name] != None): qryList.append({name:{'$gte':min(var[name])}}) qryList.append({name:{'$lte':max(var[name])}}) # construct the final query definition qryDict = {'$and': qryList} # connect to the database poesData = db.getDataConn(dbName='gme', collName='poes') # do the query qry = poesData.find(qryDict) if qryList != [] else poesData.find() if(qry.count() > 0): poesList = [] for rec in qry.sort('time'): poesList.append(poesRec(dbDict=rec)) logging.info('nreturning a list with '+ len(poesList) + ' records of poes data') return poesList # if we didn't find anything on the mongodb else: logging.info('could not find requested data in the mongodb') return None
def readAe(sTime=None, eTime=None, res=60, ae=None, al=None, au=None, ao=None): """This function reads ae data from the mongodb. **The data are 1-minute values** Parameters ---------- sTime : Optional[datetime] the earliest time you want data for, default=None eTime : Optional[datetime] the latest time you want data for. if this is None, end Time will be 1 day after sTime. default = None res : Optional[int] the time resolution desired in minutes. Valid inputs are 1 and 60. default = 60 ae : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with ae values in the range [a,b] will be returned. default = None al : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with al values in the range [a,b] will be returned. default = None au : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with au values in the range [a,b] will be returned. default = None ao : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with ao values in the range [a,b] will be returned. default = None Returns ------- aeList : list if data is found, a list of class gme.ind.ae.aeRec objects matching the input parameters is returned. If no data is found, None is returned. Example ------- import datetime as dt aeList = gme.ind.readAe(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),res=60,ao=[-50,50]) written by AJ, 20130131 """ import datetime as dt import davitpy.pydarn.sdio.dbUtils as db #check all the inputs for validity assert(sTime == None or isinstance(sTime,dt.datetime)), \ logging.error('sTime must be a datetime object') assert(eTime == None or isinstance(eTime,dt.datetime)), \ logging.error('eTime must be either None or a datetime object') assert (res == 60 or res == 1), logging.error('res must be 1 or 60') var = locals() for name in ['ae', 'al', 'au', 'ao']: assert(var[name] == None or (isinstance(var[name],list) and \ isinstance(var[name][0],(int,float)) and isinstance(var[name][1],(int,float)))), \ logging.error(name + ' must None or a list of 2 numbers') if (eTime == None and sTime != None): eTime = sTime + dt.timedelta(days=1) qryList = [] #if arguments are provided, query for those if (sTime != None): qryList.append({'time': {'$gte': sTime}}) if (eTime != None): qryList.append({'time': {'$lte': eTime}}) qryList.append({'res': res}) var = locals() for name in ['ae', 'al', 'au', 'ao']: if (var[name] != None): qryList.append({name: {'$gte': min(var[name])}}) qryList.append({name: {'$lte': max(var[name])}}) #construct the final query definition qryDict = {'$and': qryList} #connect to the database aeData = db.getDataConn(dbName='gme', collName='ae') #do the query if (qryList != []): qry = aeData.find(qryDict) else: qry = aeData.find() if (qry.count() > 0): aeList = [] for rec in qry.sort('time'): aeList.append(aeRec(dbDict=rec)) logging.info('\nreturning a list with ' + len(aeList) + 'records of ae data') return aeList #if we didn't find anything on the mongodb else: logging.info('\ncould not find requested data in the mongodb') return None
def mapOmniMongo(sYear, eYear=None, res=5): """This function reads omni data from the NASA SPDF FTP server via anonymous FTP connection and maps it to the mongodb. .. warning:: In general, nobody except the database admins will need to use this function **Args**: * **sYear** (int): the year to begin mapping data * [**eYear**] (int or None): the end year for mapping data. if this is None, eYear will be sYear * [**res**] (int): the time resolution for mapping data. Can be either 1 or 5. default=5 **Returns**: * Nothing. **Example**: :: gme.ind.mapOmniMongo(1997,res=1) written by AJ, 20130123 """ import davitpy.pydarn.sdio.dbUtils as db from davitpy import rcParams import datetime as dt #check inputs assert (res == 1 or res == 5), 'error, res must be either 1 or 5' assert (isinstance(sYear, int)), 'error, sYear must be int' if (eYear == None): eYear = sYear assert (isinstance(eYear, int)), 'error, sYear must be None or int' assert (eYear >= sYear), 'error, end year greater than start year' #get data connection mongoData = db.getDataConn(username=rcParams['DBWRITEUSER'],password=rcParams['DBWRITEPASS'],\ dbAddress=rcParams['SDDB'],dbName='gme',collName='omni') #set up all of the indices mongoData.ensure_index('time') mongoData.ensure_index('res') mongoData.ensure_index('bx') mongoData.ensure_index('bye') mongoData.ensure_index('bze') mongoData.ensure_index('bym') mongoData.ensure_index('bzm') mongoData.ensure_index('pDyn') mongoData.ensure_index('ae') mongoData.ensure_index('symh') #read the omni data from the FTP server for yr in range(sYear, eYear + 1): for mon in range(1, 13): templist = readOmniFtp(dt.datetime(yr, mon, 1), dt.datetime(yr, mon, 1) + dt.timedelta(days=31), res=res) if (templist == None): continue for rec in templist: #check if a duplicate record exists qry = mongoData.find( {'$and': [{ 'time': rec.time }, { 'res': rec.res }]}) print rec.time tempRec = rec.toDbDict() cnt = qry.count() #if this is a new record, insert it if (cnt == 0): mongoData.insert(tempRec) #if this is an existing record, update it elif (cnt == 1): print 'foundone!!' dbDict = qry.next() temp = dbDict['_id'] dbDict = tempRec dbDict['_id'] = temp mongoData.save(dbDict) else: print 'strange, there is more than 1 record for', rec.time
def readKp(sTime=None,eTime=None,kpMin=None,apMin=None,kpSum=None,apMean=None,sunspot=None): """This function reads kp data. First, it will try to get it from the mongodb, and if it can't find it, it will look on the GFZ ftp server using gme.ind.kp.readKpFtp Parameters ---------- sTime : Optional[datetime] the earliest time you want data for. if this is None, start time will be the earliest record found. default=None eTime : Optional[datetime] the latest time you want data for. if this is None, end Time will be latest record found. default=None kpMin : Optional[int] specify this to only return data from dates with a 3-hour kp value of minimum kpMin. if this is none, it will be ignored. default=None apMin : Optional[int] specify this to only return data from dates with a 3-hour ap value of minimum apMin. if this is none, it will be ignored. default=None kpSum : Optional[list] this must be a 2 element list of integers. if this is specified, only dates with kpSum values in the range [a,b] will be returned. if this is None, it will be ignored. default=None apMean : Optional[list] this must be a 2 element list of integers. if this is specified, only dates with apMean values in the range [a,b] will be returned. if this is None, it will be ignored. default=None sunspot : Optionl[list] this must be a 2 element list of integers. if this is specified, only dates with sunspot values in the range [a,b] will be returned. if this is None, it will be ignored. default=None Returns ------- kpList : list if data is found, a list of class gme.ind.kp.kpDay objects matching the input parameters is returned. If not data is found, None is returned. Example ------- import datetime as dt kpList = gme.ind.readKp(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),kpMin=2,apMin=1,kpSum=[0,10],apMean=[0,50],sunspot=[6,100]) written by AJ, 20130123 """ import datetime as dt import davitpy.pydarn.sdio.dbUtils as db #check all the inputs for validity assert(sTime == None or isinstance(sTime,dt.datetime)), \ logging.error('sTime must be either None or a datetime object') assert(eTime == None or isinstance(eTime,dt.datetime)), \ logging.error('eTime must be either None or a datetime object') assert(kpMin == None or isinstance(kpMin,int)), \ logging.error('kpMin must be either None or an int') assert(apMin == None or isinstance(apMin,int)), \ logging.error('apMin must be either None or an int') assert(kpSum == None or (isinstance(kpSum,list) and len(kpSum) == 2 and \ isinstance(kpSum[0], int) and isinstance(kpSum[1], int))), \ logging.error('kpSum must be either None or a 2 element list') assert(apMean == None or (isinstance(apMean,list) and len(apMean) == 2and \ isinstance(apMean[0], int) and isinstance(apMean[1], int))), \ logging.error('apMean must be either None or a 2 element list') assert(sunspot == None or (isinstance(sunspot,list) and len(sunspot) == 2and \ isinstance(sunspot[0], int) and isinstance(sunspot[1], int))), \ logging.error('sunspot must be either None or a 2 element list') qryList = [] #if arguments are provided, query for those if(sTime != None): qryList.append({'time':{'$gte':sTime}}) if(eTime != None): qryList.append({'time':{'$lte':eTime}}) if(kpMin != None): qryList.append({'kp':{'$gte':kpMin}}) if(apMin != None): qryList.append({'ap':{'$gte':kpMin}}) if(kpSum != None): qryList.append({'kpSum':{'$gte':min(kpSum)}}) if(kpSum != None): qryList.append({'kpSum':{'$lte':max(kpSum)}}) if(apMean != None): qryList.append({'apMean':{'$gte':min(apMean)}}) if(apMean != None): qryList.append({'apMean':{'$lte':max(apMean)}}) if(sunspot != None): qryList.append({'sunspot':{'$gte':min(sunspot)}}) if(sunspot != None): qryList.append({'sunspot':{'$lte':max(sunspot)}}) #construct the final query definition qryDict = {'$and': qryList} #connect to the database kpData = db.getDataConn(dbName='gme',collName='kp') #do the query if(qryList != []): qry = kpData.find(qryDict) else: qry = kpData.find() if(qry.count() > 0): kpList = [] for rec in qry.sort('time'): kpList.append(kpDay(dbDict=rec)) logging.info('\nreturning a list with',len(kpList),'days of kp data') return kpList #if we didn't find anything ont he mongodb else: logging.warning('\ncould not find requested data in the mongodb') logging.warning('we will look on the ftp server, but your conditions will be (mostly) ignored') if(sTime == None): logging.info('start time for search set to 1980...') sTime = dt.datetime(1980,1,1) kpList = [] if(eTime == None): eTime = dt.now() for yr in range(sTime.year,eTime.year+1): tmpList = readKpFtp(dt.datetime(yr,1,1), eTime=dt.datetime(yr,12,31)) if(tmpList == None): continue for x in tmpList: kpList.append(x) if(kpList != []): logging.info('\nreturning a list with ' + len(kpList) + ' days of kp data') return kpList else: logging.info('\n no data found on FTP server, returning None...') return None
def mapSymAsyMongo(sYear,eYear=None): """This function reads sym/asy data from wdc and puts it in mongodb .. warning:: In general, nobody except the database admins will need to use this function Parameters ---------- sYear : int the year to begin mapping data eYear : Optional[int] the end year for mapping data. if this is None, eYear will be sYear Returns ------- Nothing Example ------- gme.ind.mapSymAsyMongo(2001) written by AJ, 20130123 """ import davitpy.pydarn.sdio.dbUtils as db from davitpy import rcParams import datetime as dt #check inputs assert(isinstance(sYear,int)), logging.error('sYear must be int') if(eYear == None): eYear=sYear assert(isinstance(eYear,int)), logging.error('sYear must be None or int') assert(eYear >= sYear), logging.error('end year greater than start year') #get data connection mongoData = db.getDataConn(username=rcParams['DBWRITEUSER'],password=rcParams['DBWRITEPASS'],\ dbAddress=rcParams['SDDB'],dbName='gme',collName='symasy') #set up all of the indices mongoData.ensure_index('time') mongoData.ensure_index('symh') mongoData.ensure_index('symd') mongoData.ensure_index('asyh') mongoData.ensure_index('asyd') for yr in range(sYear,eYear+1): #1 day at a time, to not fill up RAM templist = readSymAsyWeb(dt.datetime(yr,1,1),dt.datetime(yr,1,1)+dt.timedelta(days=366)) if(templist == None): continue for rec in templist: #check if a duplicate record exists qry = mongoData.find({'time':rec.time}) logging.debug(rec.time) tempRec = rec.toDbDict() cnt = qry.count() #if this is a new record, insert it if(cnt == 0): mongoData.insert(tempRec) #if this is an existing record, update it elif(cnt == 1): logging.debug('found one!!') dbDict = qry.next() temp = dbDict['_id'] dbDict = tempRec dbDict['_id'] = temp mongoData.save(dbDict) else: logging.warning('strange, there is more than 1 Sym/Asy record for' + rec.time) del templist
def readSymAsy(sTime=None,eTime=None,symh=None,symd=None,asyh=None,asyd=None): """This function reads sym/asy data from the mongodb. Parameters ---------- sTime : Optional[datetime] the earliest time you want data for, default=None eTime : Optional[datetime] the latest time you want data for. if this is None, end Time will be 1 day after sTime. default = None symh : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with symh values in the range [a,b] will be returned. default = None symd : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with symd values in the range [a,b] will be returned. default = None asyh : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with asyh values in the range [a,b] will be returned. default = None asyd : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with asyd values in the range [a,b] will be returned. default = None Returns ------- symList : list if data is found, a list of class gme.ind.symasy.symAsyRec objects matching the input parameters is returned. If no data is found, None is returned. Example ------- import datetime as dt symList = gme.ind.readSymAsy(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),symh=[5,50],asyd=[-10,0]) written by AJ, 20130131 """ import datetime as dt import davitpy.pydarn.sdio.dbUtils as db #check all the inputs for validity assert(sTime == None or isinstance(sTime,dt.datetime)), \ logging.error('sTime must be a datetime object') assert(eTime == None or isinstance(eTime,dt.datetime)), \ logging.error('eTime must be either None or a datetime object') var = locals() for name in ['symh','symd','asyh','asyd']: assert(var[name] == None or (isinstance(var[name],list) and \ isinstance(var[name][0],(int,float)) and isinstance(var[name][1],(int,float)))), \ logging.error(name + ' must None or a list of 2 numbers') if(eTime == None and sTime != None): eTime = sTime+dt.timedelta(days=1) qryList = [] #if arguments are provided, query for those if(sTime != None): qryList.append({'time':{'$gte':sTime}}) if(eTime != None): qryList.append({'time':{'$lte':eTime}}) var = locals() for name in ['symh','symd','asyh','asyd']: if(var[name] != None): qryList.append({name:{'$gte':min(var[name])}}) qryList.append({name:{'$lte':max(var[name])}}) #construct the final query definition qryDict = {'$and': qryList} #connect to the database symData = db.getDataConn(dbName='gme',collName='symasy') #do the query if(qryList != []): qry = symData.find(qryDict) else: qry = symData.find() if(qry.count() > 0): symList = [] for rec in qry.sort('time'): symList.append(symAsyRec(dbDict=rec)) logging.info('\nreturning a list with ' + str(len(symList)) + ' records of sym/asy data') return symList #if we didn't find anything on the mongodb else: logging.info('\ncould not find requested data in the mongodb') return None
def readKp(sTime=None, eTime=None, kpMin=None, apMin=None, kpSum=None, apMean=None, sunspot=None): """This function reads kp data. First, it will try to get it from the mongodb, and if it can't find it, it will look on the GFZ ftp server using gme.ind.kp.readKpFtp Parameters ---------- sTime : Optional[datetime] the earliest time you want data for. if this is None, start time will be the earliest record found. default=None eTime : Optional[datetime] the latest time you want data for. if this is None, end Time will be latest record found. default=None kpMin : Optional[int] specify this to only return data from dates with a 3-hour kp value of minimum kpMin. if this is none, it will be ignored. default=None apMin : Optional[int] specify this to only return data from dates with a 3-hour ap value of minimum apMin. if this is none, it will be ignored. default=None kpSum : Optional[list] this must be a 2 element list of integers. if this is specified, only dates with kpSum values in the range [a,b] will be returned. if this is None, it will be ignored. default=None apMean : Optional[list] this must be a 2 element list of integers. if this is specified, only dates with apMean values in the range [a,b] will be returned. if this is None, it will be ignored. default=None sunspot : Optionl[list] this must be a 2 element list of integers. if this is specified, only dates with sunspot values in the range [a,b] will be returned. if this is None, it will be ignored. default=None Returns ------- kpList : list if data is found, a list of class gme.ind.kp.kpDay objects matching the input parameters is returned. If not data is found, None is returned. Example ------- import datetime as dt kpList = gme.ind.readKp(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),kpMin=2,apMin=1,kpSum=[0,10],apMean=[0,50],sunspot=[6,100]) written by AJ, 20130123 """ import datetime as dt import davitpy.pydarn.sdio.dbUtils as db #check all the inputs for validity assert(sTime == None or isinstance(sTime,dt.datetime)), \ logging.error('sTime must be either None or a datetime object') assert(eTime == None or isinstance(eTime,dt.datetime)), \ logging.error('eTime must be either None or a datetime object') assert(kpMin == None or isinstance(kpMin,int)), \ logging.error('kpMin must be either None or an int') assert(apMin == None or isinstance(apMin,int)), \ logging.error('apMin must be either None or an int') assert(kpSum == None or (isinstance(kpSum,list) and len(kpSum) == 2 and \ isinstance(kpSum[0], int) and isinstance(kpSum[1], int))), \ logging.error('kpSum must be either None or a 2 element list') assert(apMean == None or (isinstance(apMean,list) and len(apMean) == 2and \ isinstance(apMean[0], int) and isinstance(apMean[1], int))), \ logging.error('apMean must be either None or a 2 element list') assert(sunspot == None or (isinstance(sunspot,list) and len(sunspot) == 2and \ isinstance(sunspot[0], int) and isinstance(sunspot[1], int))), \ logging.error('sunspot must be either None or a 2 element list') qryList = [] #if arguments are provided, query for those if (sTime != None): qryList.append({'time': {'$gte': sTime}}) if (eTime != None): qryList.append({'time': {'$lte': eTime}}) if (kpMin != None): qryList.append({'kp': {'$gte': kpMin}}) if (apMin != None): qryList.append({'ap': {'$gte': kpMin}}) if (kpSum != None): qryList.append({'kpSum': {'$gte': min(kpSum)}}) if (kpSum != None): qryList.append({'kpSum': {'$lte': max(kpSum)}}) if (apMean != None): qryList.append({'apMean': {'$gte': min(apMean)}}) if (apMean != None): qryList.append({'apMean': {'$lte': max(apMean)}}) if (sunspot != None): qryList.append({'sunspot': {'$gte': min(sunspot)}}) if (sunspot != None): qryList.append({'sunspot': {'$lte': max(sunspot)}}) #construct the final query definition qryDict = {'$and': qryList} #connect to the database kpData = db.getDataConn(dbName='gme', collName='kp') #do the query if (qryList != []): qry = kpData.find(qryDict) else: qry = kpData.find() if (qry.count() > 0): kpList = [] for rec in qry.sort('time'): kpList.append(kpDay(dbDict=rec)) logging.info('\nreturning a list with', len(kpList), 'days of kp data') return kpList #if we didn't find anything ont he mongodb else: logging.warning('\ncould not find requested data in the mongodb') logging.warning( 'we will look on the ftp server, but your conditions will be (mostly) ignored' ) if (sTime == None): logging.info('start time for search set to 1980...') sTime = dt.datetime(1980, 1, 1) kpList = [] if (eTime == None): eTime = dt.now() for yr in range(sTime.year, eTime.year + 1): tmpList = readKpFtp(dt.datetime(yr, 1, 1), eTime=dt.datetime(yr, 12, 31)) if (tmpList == None): continue for x in tmpList: kpList.append(x) if (kpList != []): logging.info('\nreturning a list with ' + str(len(kpList)) + ' days of kp data') return kpList else: logging.info('\n no data found on FTP server, returning None...') return None
def readSymAsy(sTime=None, eTime=None, symh=None, symd=None, asyh=None, asyd=None): """This function reads sym/asy data from the mongodb. Parameters ---------- sTime : Optional[datetime] the earliest time you want data for, default=None eTime : Optional[datetime] the latest time you want data for. if this is None, end Time will be 1 day after sTime. default = None symh : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with symh values in the range [a,b] will be returned. default = None symd : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with symd values in the range [a,b] will be returned. default = None asyh : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with asyh values in the range [a,b] will be returned. default = None asyd : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with asyd values in the range [a,b] will be returned. default = None Returns ------- symList : list if data is found, a list of class gme.ind.symasy.symAsyRec objects matching the input parameters is returned. If no data is found, None is returned. Example ------- import datetime as dt symList = gme.ind.readSymAsy(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),symh=[5,50],asyd=[-10,0]) written by AJ, 20130131 """ import datetime as dt import davitpy.pydarn.sdio.dbUtils as db #check all the inputs for validity assert(sTime == None or isinstance(sTime,dt.datetime)), \ logging.error('sTime must be a datetime object') assert(eTime == None or isinstance(eTime,dt.datetime)), \ logging.error('eTime must be either None or a datetime object') var = locals() for name in ['symh', 'symd', 'asyh', 'asyd']: assert(var[name] == None or (isinstance(var[name],list) and \ isinstance(var[name][0],(int,float)) and isinstance(var[name][1],(int,float)))), \ logging.error(name + ' must None or a list of 2 numbers') if (eTime == None and sTime != None): eTime = sTime + dt.timedelta(days=1) qryList = [] #if arguments are provided, query for those if (sTime != None): qryList.append({'time': {'$gte': sTime}}) if (eTime != None): qryList.append({'time': {'$lte': eTime}}) var = locals() for name in ['symh', 'symd', 'asyh', 'asyd']: if (var[name] != None): qryList.append({name: {'$gte': min(var[name])}}) qryList.append({name: {'$lte': max(var[name])}}) #construct the final query definition qryDict = {'$and': qryList} #connect to the database symData = db.getDataConn(dbName='gme', collName='symasy') #do the query if (qryList != []): qry = symData.find(qryDict) else: qry = symData.find() if (qry.count() > 0): symList = [] for rec in qry.sort('time'): symList.append(symAsyRec(dbDict=rec)) logging.info('\nreturning a list with ' + str(len(symList)) + ' records of sym/asy data') return symList #if we didn't find anything on the mongodb else: logging.info('\ncould not find requested data in the mongodb') return None
def readOmni(sTime, eTime=None, res=5, bx=None, bye=None, bze=None, bym=None, bzm=None, pDyn=None, ae=None, symh=None): """This function reads omni data. First, it will try to get it from the mongodb, and if it can't find it, it will look on the NASA SPDF FTP server using readOmniFtp Parameters ---------- sTime : datetime the earliest time you want data for eTime : Optional[datetime] the latest time you want data for. if this is None, end Time will be 1 day after sTime. default = None res : Optional[int] the time reolution of data desired. This can be either 1 or 5. default = 5 bx : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bx values in the range [a,b] will be returned. default = None bx : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bx values in the range [a,b] will be returned. default = None bye : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bye values in the range [a,b] will be returned. default = None bze : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bze values in the range [a,b] will be returned. default = None bym : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bym values in the range [a,b] will be returned. default = None bzm : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bzm values in the range [a,b] will be returned. default = None pDyn : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with pDyn values in the range [a,b] will be returned. default = None ae : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with ae values in the range [a,b] will be returned. default = None symh : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with symh values in the range [a,b] will be returned. default = None Returns ------- omniList : list if data is found, a list of class omniRec objects matching the input parameters is returned. If no data is found, None is returned. Example ------- import datetime as dt omniList = gme.ind.readOmni(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),bx=[0,5.5],bye=[-1,3.5],bze=[-10,0],ae=[0,56.3]) written by AJ, 20130128 """ import datetime as dt import davitpy.pydarn.sdio.dbUtils as db #check all the inputs for validity assert(isinstance(sTime,dt.datetime)), \ logging.error('sTime must be a datetime object') assert(eTime == None or isinstance(eTime,dt.datetime)), \ logging.error('eTime must be either None or a datetime object') assert (res == 5 or res == 1), logging.error('res must be either 1 or 5') var = locals() for name in ['bx', 'bye', 'bze', 'bym', 'bzm', 'pDyn', 'ae', 'symh']: assert(var[name] == None or (isinstance(var[name],list) and \ isinstance(var[name][0],(int,float)) and isinstance(var[name][1],(int,float)))), \ logging.error(name + ' must None or a list of 2 numbers') if (eTime == None): eTime = sTime + dt.timedelta(days=1) qryList = [] #if arguments are provided, query for those qryList.append({'time': {'$gte': sTime}}) qryList.append({'res': res}) if (eTime != None): qryList.append({'time': {'$lte': eTime}}) var = locals() for name in ['bx', 'bye', 'bze', 'bym', 'bzm', 'pDyn', 'ae', 'symh']: if (var[name] != None): qryList.append({name: {'$gte': min(var[name])}}) qryList.append({name: {'$lte': max(var[name])}}) #construct the final query definition qryDict = {'$and': qryList} #connect to the database omniData = db.getDataConn(dbName='gme', collName='omni') #do the query if (qryList != []): qry = omniData.find(qryDict) else: qry = omniData.find() if (qry.count() > 0): omniList = [] for rec in qry.sort('time'): omniList.append(omniRec(dbDict=rec)) logging.info('\nreturning a list with ' + len(omniList) + ' records of omni data') return omniList #if we didn't find anything on the mongodb else: logging.info('\ncould not find requested data in the mongodb') logging.info( 'we will look on the ftp server, but your conditions will be (mostly) ignored' ) #read from ftp server omniList = readOmniFtp(sTime, eTime) if (omniList != None): logging.info('\nreturning a list with ' + len(omniList) + ' recs of omni data') return omniList else: logging.info('\n no data found on FTP server, returning None...') return None
def readAe(sTime=None, eTime=None, res=60, ae=None, al=None, au=None, ao=None): """This function reads ae data from the mongodb. **The data are 1-minute values** Parameters ---------- sTime : Optional[datetime] the earliest time you want data for, default=None eTime : Optional[datetime] the latest time you want data for. if this is None, end Time will be 1 day after sTime. default = None res : Optional[int] the time resolution desired in minutes. Valid inputs are 1 and 60. default = 60 ae : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with ae values in the range [a,b] will be returned. default = None al : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with al values in the range [a,b] will be returned. default = None au : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with au values in the range [a,b] will be returned. default = None ao : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with ao values in the range [a,b] will be returned. default = None Returns ------- aeList : list if data is found, a list of class gme.ind.ae.aeRec objects matching the input parameters is returned. If no data is found, None is returned. Example ------- import datetime as dt aeList = gme.ind.readAe(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),res=60,ao=[-50,50]) written by AJ, 20130131 """ import datetime as dt import davitpy.pydarn.sdio.dbUtils as db # check all the inputs for validity assert sTime == None or isinstance(sTime, dt.datetime), logging.error("sTime must be a datetime object") assert eTime == None or isinstance(eTime, dt.datetime), logging.error( "eTime must be either None or a datetime object" ) assert res == 60 or res == 1, logging.error("res must be 1 or 60") var = locals() for name in ["ae", "al", "au", "ao"]: assert var[name] == None or ( isinstance(var[name], list) and isinstance(var[name][0], (int, float)) and isinstance(var[name][1], (int, float)) ), logging.error(name + " must None or a list of 2 numbers") if eTime == None and sTime != None: eTime = sTime + dt.timedelta(days=1) qryList = [] # if arguments are provided, query for those if sTime != None: qryList.append({"time": {"$gte": sTime}}) if eTime != None: qryList.append({"time": {"$lte": eTime}}) qryList.append({"res": res}) var = locals() for name in ["ae", "al", "au", "ao"]: if var[name] != None: qryList.append({name: {"$gte": min(var[name])}}) qryList.append({name: {"$lte": max(var[name])}}) # construct the final query definition qryDict = {"$and": qryList} # connect to the database aeData = db.getDataConn(dbName="gme", collName="ae") # do the query if qryList != []: qry = aeData.find(qryDict) else: qry = aeData.find() if qry.count() > 0: aeList = [] for rec in qry.sort("time"): aeList.append(aeRec(dbDict=rec)) logging.info("\nreturning a list with " + str(len(aeList)) + " records of ae data") return aeList # if we didn't find anything on the mongodb else: logging.info("\ncould not find requested data in the mongodb") return None
def readOmni(sTime, eTime=None, res=5, bx=None, bye=None, bze=None, bym=None, bzm=None, pDyn=None, ae=None, symh=None): """This function reads omni data. First, it will try to get it from the mongodb, and if it can't find it, it will look on the NASA SPDF FTP server using :func:`readOmniFtp` **Args**: * **sTime** (`datetime <http://tinyurl.com/bl352yx>`_ or None): the earliest time you want data for * [**eTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the latest time you want data for. if this is None, end Time will be 1 day after sTime. default = None * [**res**] (int): the time reolution of data desired. This can be either 1 or 5. default = 5 * [**bx**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bx values in the range [a,b] will be returned. default = None * [**bx**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bx values in the range [a,b] will be returned. default = None * [**bye**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bye values in the range [a,b] will be returned. default = None * [**bze**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bze values in the range [a,b] will be returned. default = None * [**bym**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bym values in the range [a,b] will be returned. default = None * [**bzm**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bzm values in the range [a,b] will be returned. default = None * [**pDyn**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with pDyn values in the range [a,b] will be returned. default = None * [**ae**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with ae values in the range [a,b] will be returned. default = None * [**symh**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with symh values in the range [a,b] will be returned. default = None **Returns**: * **omniList** (list or None): if data is found, a list of :class:`omniRec` objects matching the input parameters is returned. If no data is found, None is returned. **Example**: :: import datetime as dt omniList = gme.ind.readOmni(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),bx=[0,5.5],bye=[-1,3.5],bze=[-10,0],ae=[0,56.3]) written by AJ, 20130128 """ import datetime as dt import davitpy.pydarn.sdio.dbUtils as db #check all the inputs for validity assert(isinstance(sTime,dt.datetime)), \ 'error, sTime must be a datetime object' assert(eTime == None or isinstance(eTime,dt.datetime)), \ 'error, eTime must be either None or a datetime object' assert (res == 5 or res == 1), 'error, res must be either 1 or 5' var = locals() for name in ['bx', 'bye', 'bze', 'bym', 'bzm', 'pDyn', 'ae', 'symh']: assert(var[name] == None or (isinstance(var[name],list) and \ isinstance(var[name][0],(int,float)) and isinstance(var[name][1],(int,float)))), \ 'error,'+name+' must None or a list of 2 numbers' if (eTime == None): eTime = sTime + dt.timedelta(days=1) qryList = [] #if arguments are provided, query for those qryList.append({'time': {'$gte': sTime}}) qryList.append({'res': res}) if (eTime != None): qryList.append({'time': {'$lte': eTime}}) var = locals() for name in ['bx', 'bye', 'bze', 'bym', 'bzm', 'pDyn', 'ae', 'symh']: if (var[name] != None): qryList.append({name: {'$gte': min(var[name])}}) qryList.append({name: {'$lte': max(var[name])}}) #construct the final query definition qryDict = {'$and': qryList} #connect to the database omniData = db.getDataConn(dbName='gme', collName='omni') #do the query if (qryList != []): qry = omniData.find(qryDict) else: qry = omniData.find() if (qry.count() > 0): omniList = [] for rec in qry.sort('time'): omniList.append(omniRec(dbDict=rec)) print '\nreturning a list with', len(omniList), 'records of omni data' return omniList #if we didn't find anything on the mongodb else: print '\ncould not find requested data in the mongodb' print 'we will look on the ftp server, but your conditions will be (mostly) ignored' #read from ftp server omniList = readOmniFtp(sTime, eTime) if (omniList != None): print '\nreturning a list with', len(omniList), 'recs of omni data' return omniList else: print '\n no data found on FTP server, returning None...' return None
def mapAeMongo(sYear, eYear=None, res=60): """This function reads ae data from wdc and puts it in mongodb Parameters ---------- sYear : int the year to begin mapping data eYear : Optional[int] the end year for mapping data. if this is None, eYear will be sYear res : Optional[int] the time resolution desired. either 1 or 60 minutes. default=60. Returns ------- Nothing Notes ----- In general, nobody except the database admins will need to use this function Example ------- gme.ind.mapAeMongo(1997) written by AJ, 20130123 """ import davitpy.pydarn.sdio.dbUtils as db from davitpy import rcParams import datetime as dt # check inputs assert isinstance(sYear, int), logging.error("sYear must be int") if eYear == None: eYear = sYear assert isinstance(eYear, int), logging.error("sYear must be None or int") assert eYear >= sYear, logging.error("end year less than than start year") # get data connection mongoData = db.getDataConn( username=rcParams["DBWRITEUSER"], password=rcParams["DBWRITEPASS"], dbAddress=rcParams["SDDB"], dbName="gme", collName="ae", ) # set up all of the indices mongoData.ensure_index("time") mongoData.ensure_index("ae") mongoData.ensure_index("al") mongoData.ensure_index("au") mongoData.ensure_index("ao") mongoData.ensure_index("res") for yr in range(sYear, eYear + 1): # 1 day at a time, to not fill up RAM templist = readAeWeb(dt.datetime(yr, 1, 1), dt.datetime(yr, 1, 1) + dt.timedelta(days=366), res=res) if templist == None: continue for rec in templist: # check if a duplicate record exists qry = mongoData.find({"$and": [{"time": rec.time}, {"res": rec.res}]}) logging.debug(rec.time) tempRec = rec.toDbDict() cnt = qry.count() # if this is a new record, insert it if cnt == 0: mongoData.insert(tempRec) # if this is an existing record, update it elif cnt == 1: logging.debug("foundone!!") dbDict = qry.next() temp = dbDict["_id"] dbDict = tempRec dbDict["_id"] = temp mongoData.save(dbDict) else: logging.warning("strange, there is more than 1 AE record for " + rec.time) del templist
def readAe(sTime=None,eTime=None,res=60,ae=None,al=None,au=None,ao=None): """This function reads ae data from the mongodb. **The data are 1-minute values** **Args**: * [**sTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the earliest time you want data for, default=None * [**eTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the latest time you want data for. if this is None, end Time will be 1 day after sTime. default = None * [**res**] (int): the time resolution desired in minutes. Valid inputs are 1 and 60. default = 60 * [**ae**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with ae values in the range [a,b] will be returned. default = None * [**al**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with al values in the range [a,b] will be returned. default = None * [**au**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with au values in the range [a,b] will be returned. default = None * [**ao**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with ao values in the range [a,b] will be returned. default = None **Returns**: * **aeList** (list or None): if data is found, a list of :class:`gme.ind.ae.aeRec` objects matching the input parameters is returned. If no data is found, None is returned. **Example**: :: import datetime as dt aeList = gme.ind.readAe(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),res=60,ao=[-50,50]) written by AJ, 20130131 """ import datetime as dt import davitpy.pydarn.sdio.dbUtils as db #check all the inputs for validity assert(sTime == None or isinstance(sTime,dt.datetime)), \ 'error, sTime must be a datetime object' assert(eTime == None or isinstance(eTime,dt.datetime)), \ 'error, eTime must be either None or a datetime object' assert(res == 60 or res ==1), 'error, res must be 1 or 60' var = locals() for name in ['ae','al','au','ao']: assert(var[name] == None or (isinstance(var[name],list) and \ isinstance(var[name][0],(int,float)) and isinstance(var[name][1],(int,float)))), \ 'error,'+name+' must None or a list of 2 numbers' if(eTime == None and sTime != None): eTime = sTime+dt.timedelta(days=1) qryList = [] #if arguments are provided, query for those if(sTime != None): qryList.append({'time':{'$gte':sTime}}) if(eTime != None): qryList.append({'time':{'$lte':eTime}}) qryList.append({'res':res}) var = locals() for name in ['ae','al','au','ao']: if(var[name] != None): qryList.append({name:{'$gte':min(var[name])}}) qryList.append({name:{'$lte':max(var[name])}}) #construct the final query definition qryDict = {'$and': qryList} #connect to the database aeData = db.getDataConn(dbName='gme',collName='ae') #do the query if(qryList != []): qry = aeData.find(qryDict) else: qry = aeData.find() if(qry.count() > 0): aeList = [] for rec in qry.sort('time'): aeList.append(aeRec(dbDict=rec)) print '\nreturning a list with',len(aeList),'records of ae data' return aeList #if we didn't find anything on the mongodb else: print '\ncould not find requested data in the mongodb' return None
def readOmni(sTime,eTime=None,res=5,bx=None,bye=None,bze=None,bym=None,bzm=None,pDyn=None,ae=None,symh=None): """This function reads omni data. First, it will try to get it from the mongodb, and if it can't find it, it will look on the NASA SPDF FTP server using readOmniFtp Parameters ---------- sTime : datetime the earliest time you want data for eTime : Optional[datetime] the latest time you want data for. if this is None, end Time will be 1 day after sTime. default = None res : Optional[int] the time reolution of data desired. This can be either 1 or 5. default = 5 bx : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bx values in the range [a,b] will be returned. default = None bx : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bx values in the range [a,b] will be returned. default = None bye : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bye values in the range [a,b] will be returned. default = None bze : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bze values in the range [a,b] will be returned. default = None bym : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bym values in the range [a,b] will be returned. default = None bzm : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bzm values in the range [a,b] will be returned. default = None pDyn : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with pDyn values in the range [a,b] will be returned. default = None ae : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with ae values in the range [a,b] will be returned. default = None symh : Optional[list] if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with symh values in the range [a,b] will be returned. default = None Returns ------- omniList : list if data is found, a list of class omniRec objects matching the input parameters is returned. If no data is found, None is returned. Example ------- import datetime as dt omniList = gme.ind.readOmni(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),bx=[0,5.5],bye=[-1,3.5],bze=[-10,0],ae=[0,56.3]) written by AJ, 20130128 """ import datetime as dt import davitpy.pydarn.sdio.dbUtils as db #check all the inputs for validity assert(isinstance(sTime,dt.datetime)), \ logging.error('sTime must be a datetime object') assert(eTime == None or isinstance(eTime,dt.datetime)), \ logging.error('eTime must be either None or a datetime object') assert(res==5 or res == 1), logging.error('res must be either 1 or 5') var = locals() for name in ['bx','bye','bze','bym','bzm','pDyn','ae','symh']: assert(var[name] == None or (isinstance(var[name],list) and \ isinstance(var[name][0],(int,float)) and isinstance(var[name][1],(int,float)))), \ logging.error(name + ' must None or a list of 2 numbers') if(eTime == None): eTime = sTime+dt.timedelta(days=1) qryList = [] #if arguments are provided, query for those qryList.append({'time':{'$gte':sTime}}) qryList.append({'res':res}) if(eTime != None): qryList.append({'time':{'$lte':eTime}}) var = locals() for name in ['bx','bye','bze','bym','bzm','pDyn','ae','symh']: if(var[name] != None): qryList.append({name:{'$gte':min(var[name])}}) qryList.append({name:{'$lte':max(var[name])}}) #construct the final query definition qryDict = {'$and': qryList} #connect to the database omniData = db.getDataConn(dbName='gme',collName='omni') #do the query if(qryList != []): qry = omniData.find(qryDict) else: qry = omniData.find() if(qry.count() > 0): omniList = [] for rec in qry.sort('time'): omniList.append(omniRec(dbDict=rec)) logging.info('\nreturning a list with ' + len(omniList) + ' records of omni data') return omniList #if we didn't find anything on the mongodb else: logging.info('\ncould not find requested data in the mongodb') logging.info('we will look on the ftp server, but your conditions will be (mostly) ignored') #read from ftp server omniList = readOmniFtp(sTime, eTime) if(omniList != None): logging.info('\nreturning a list with ' + len(omniList) + ' recs of omni data') return omniList else: logging.info('\n no data found on FTP server, returning None...') return None
def mapAeMongo(sYear, eYear=None, res=60): """This function reads ae data from wdc and puts it in mongodb Parameters ---------- sYear : int the year to begin mapping data eYear : Optional[int] the end year for mapping data. if this is None, eYear will be sYear res : Optional[int] the time resolution desired. either 1 or 60 minutes. default=60. Returns ------- Nothing Notes ----- In general, nobody except the database admins will need to use this function Example ------- gme.ind.mapAeMongo(1997) written by AJ, 20130123 """ import davitpy.pydarn.sdio.dbUtils as db from davitpy import rcParams import datetime as dt #check inputs assert (isinstance(sYear, int)), logging.error('sYear must be int') if (eYear == None): eYear = sYear assert (isinstance(eYear, int)), logging.error('sYear must be None or int') assert (eYear >= sYear), logging.error('end year less than than start year') #get data connection mongoData = db.getDataConn(username=rcParams['DBWRITEUSER'],password=rcParams['DBWRITEPASS'],\ dbAddress=rcParams['SDDB'],dbName='gme',collName='ae') #set up all of the indices mongoData.ensure_index('time') mongoData.ensure_index('ae') mongoData.ensure_index('al') mongoData.ensure_index('au') mongoData.ensure_index('ao') mongoData.ensure_index('res') for yr in range(sYear, eYear + 1): #1 day at a time, to not fill up RAM templist = readAeWeb(dt.datetime(yr, 1, 1), dt.datetime(yr, 1, 1) + dt.timedelta(days=366), res=res) if (templist == None): continue for rec in templist: #check if a duplicate record exists qry = mongoData.find( {'$and': [{ 'time': rec.time }, { 'res': rec.res }]}) logging.debug(rec.time) tempRec = rec.toDbDict() cnt = qry.count() #if this is a new record, insert it if (cnt == 0): mongoData.insert(tempRec) #if this is an existing record, update it elif (cnt == 1): logging.debug('foundone!!') dbDict = qry.next() temp = dbDict['_id'] dbDict = tempRec dbDict['_id'] = temp mongoData.save(dbDict) else: logging.warning( 'strange, there is more than 1 AE record for ' + rec.time) del templist
def readPoes(sTime,eTime=None,satnum=None,folat=None,folon=None,ted=None,echar=None,pchar=None): """This function reads poes data. First, it will try to get it from the mongodb, and if it can't find it, it will look on the NOAA NGDC FTP server using :func:`readPoesFtp`. The data are 16-second averages **Args**: * **sTime** (`datetime <http://tinyurl.com/bl352yx>`_ or None): the earliest time you want data for * [**eTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the latest time you want data for. if this is None, end Time will be 1 day after sTime. default = None * [**satnum**] (int): the satellite you want data for. eg 17 for noaa17. if this is None, data for all satellites will be returned. default = None * [**satnum**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bx values in the range [a,b] will be returned. default = None * [**folat**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bx values in the range [a,b] will be returned. default = None * [**folon**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bye values in the range [a,b] will be returned. default = None * [**ted**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bze values in the range [a,b] will be returned. default = None * [**echar**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bym values in the range [a,b] will be returned. default = None * [**pchar**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bzm values in the range [a,b] will be returned. default = None **Returns**: * **poesList** (list or None): if data is found, a list of :class:`poesRec` objects matching the input parameters is returned. If no data is found, None is returned. **Example**: :: import datetime as dt poesList = gme.sat.readPoes(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),folat=[60,80]) written by AJ, 20130131 """ import datetime as dt import davitpy.pydarn.sdio.dbUtils as db #check all the inputs for validity assert(isinstance(sTime,dt.datetime)), \ 'error, sTime must be a datetime object' assert(eTime == None or isinstance(eTime,dt.datetime)), \ 'error, eTime must be either None or a datetime object' assert(satnum == None or isinstance(satnum,int)), 'error, satnum must be an int' var = locals() for name in ['folat','folon','ted','echar','pchar']: assert(var[name] == None or (isinstance(var[name],list) and \ isinstance(var[name][0],(int,float)) and isinstance(var[name][1],(int,float)))), \ 'error,'+name+' must None or a list of 2 numbers' if(eTime == None): eTime = sTime+dt.timedelta(days=1) qryList = [] #if arguments are provided, query for those qryList.append({'time':{'$gte':sTime}}) if(eTime != None): qryList.append({'time':{'$lte':eTime}}) if(satnum != None): qryList.append({'satnum':satnum}) var = locals() for name in ['folat','folon','ted','echar','pchar']: if(var[name] != None): qryList.append({name:{'$gte':min(var[name])}}) qryList.append({name:{'$lte':max(var[name])}}) #construct the final query definition qryDict = {'$and': qryList} #connect to the database poesData = db.getDataConn(dbName='gme',collName='poes') #do the query if(qryList != []): qry = poesData.find(qryDict) else: qry = poesData.find() if(qry.count() > 0): poesList = [] for rec in qry.sort('time'): poesList.append(poesRec(dbDict=rec)) print '\nreturning a list with',len(poesList),'records of poes data' return poesList #if we didn't find anything on the mongodb else: print '\ncould not find requested data in the mongodb' return None
def mapPoesMongo(syear, eYear=None): """This function reads poes data from the NOAA NGDC FTP server via anonymous FTP connection and maps it to the mongodb. Parameters ---------- syear : (int) the year to begin mapping data eYear : (int or None) the end year for mapping data. if this is None, eYear will be syear Returns ------- Nothing. Notes ----- In general, nobody except the database admins will need to use this function Example ------- gme.sat.mapPoesMongo(2004) written by AJ, 20130131 """ import davitpy.pydarn.sdio.dbUtils as db from davitpy import rcParams import datetime as dt # check inputs assert isinstance(syear, int), logging.error('syear must be int') if eYear == None: eYear = syear assert isinstance(eYear, int), logging.error('syear must be None or int') assert eYear >= syear, logging.error('end year greater than start year') # get data connection mongo_data = db.getDataConn(username=rcParams['DBWRITEUSER'], password=rcParams['DBWRITEPASS'], dbAddress=rcParams['SDDB'], dbName='gme', collName='poes') # set up all of the indices mongo_data.ensure_index('time') mongo_data.ensure_index('satnum') mongo_data.ensure_index('folat') mongo_data.ensure_index('folon') mongo_data.ensure_index('ted') mongo_data.ensure_index('echar') mongo_data.ensure_index('pchar') # read the poes data from the FTP server my_time = dt.datetime(syear, 1, 1) while my_time < dt.datetime(eYear+1, 1, 1): # 10 day at a time, to not fill up RAM templist = readPoesFtp(my_time, my_time+dt.timedelta(days=10)) if(templist == None): continue for rec in templist: # check if a duplicate record exists qry = mongo_data.find({'$and':[{'time':rec.time}, {'satnum':rec.satnum}]}) logging.info("{:}, {:}".format(rec.time, rec.satnum)) tempRec = rec.toDbDict() cnt = qry.count() # if this is a new record, insert it; if it exists, update it if cnt == 0: mongo_data.insert(tempRec) elif cnt == 1: logging.debug('found one!!') db_dict = qry.next() temp = db_dict['_id'] db_dict = tempRec db_dict['_id'] = temp mongo_data.save(db_dict) else: logging.info('strange, there is more than 1 record for ' + rec.time) del templist my_time += dt.timedelta(days=10)
def mapPoesMongo(sYear,eYear=None): """This function reads poes data from the NOAA NGDC FTP server via anonymous FTP connection and maps it to the mongodb. .. warning:: In general, nobody except the database admins will need to use this function **Args**: * **sYear** (int): the year to begin mapping data * [**eYear**] (int or None): the end year for mapping data. if this is None, eYear will be sYear **Returns**: * Nothing. **Example**: :: gme.sat.mapPoesMongo(2004) written by AJ, 20130131 """ import davitpy.pydarn.sdio.dbUtils as db from davitpy import rcParams import datetime as dt #check inputs assert(isinstance(sYear,int)),'error, sYear must be int' if(eYear == None): eYear=sYear assert(isinstance(eYear,int)),'error, sYear must be None or int' assert(eYear >= sYear), 'error, end year greater than start year' #get data connection mongoData = db.getDataConn(username=rcParams['DBWRITEUSER'],password=rcParams['DBWRITEPASS'],\ dbAddress=rcParams['SDDB'],dbName='gme',collName='poes') #set up all of the indices mongoData.ensure_index('time') mongoData.ensure_index('satnum') mongoData.ensure_index('folat') mongoData.ensure_index('folon') mongoData.ensure_index('ted') mongoData.ensure_index('echar') mongoData.ensure_index('pchar') #read the poes data from the FTP server myTime = dt.datetime(sYear,1,1) while(myTime < dt.datetime(eYear+1,1,1)): #10 day at a time, to not fill up RAM templist = readPoesFtp(myTime,myTime+dt.timedelta(days=10)) if(templist == None): continue for rec in templist: #check if a duplicate record exists qry = mongoData.find({'$and':[{'time':rec.time},{'satnum':rec.satnum}]}) print rec.time, rec.satnum tempRec = rec.toDbDict() cnt = qry.count() #if this is a new record, insert it if(cnt == 0): mongoData.insert(tempRec) #if this is an existing record, update it elif(cnt == 1): print 'foundone!!' dbDict = qry.next() temp = dbDict['_id'] dbDict = tempRec dbDict['_id'] = temp mongoData.save(dbDict) else: print 'strange, there is more than 1 record for',rec.time del templist myTime += dt.timedelta(days=10)