def mapDstMongo(sYear, eYear=None):
"""This function reads dst data from wdc and puts it in mongodb
.. warning::
In general, nobody except the database admins will need to use this function
**Args**:
* **sYear** (int): the year to begin mapping data
* [**eYear**] (int or None): the end year for mapping data. if this is None, eYear will be sYear
**Returns**:
* Nothing.
**Example**:
::
gme.ind.mapDstMongo(1997)
written by AJ, 20130123
"""
import davitpy.pydarn.sdio.dbUtils as db
from davitpy import rcParams
import datetime as dt
#check inputs
assert (isinstance(sYear, int)), 'error, sYear must be int'
if (eYear == None): eYear = sYear
assert (isinstance(eYear, int)), 'error, sYear must be None or int'
assert (eYear >= sYear), 'error, end year greater than start year'
#get data connection
mongoData = db.getDataConn(username=rcParams['DBWRITEUSER'],password=rcParams['DBWRITEPASS'],\
dbAddress=rcParams['SDDB'],dbName='gme',collName='dst')
#set up all of the indices
mongoData.ensure_index('time')
mongoData.ensure_index('dst')
for yr in range(sYear, eYear + 1):
#1 year at a time, to not fill up RAM
templist = readDstWeb(dt.datetime(yr, 1, 1), dt.datetime(yr, 12, 31))
for rec in templist:
#check if a duplicate record exists
qry = mongoData.find({'time': rec.time})
print rec.time
tempRec = rec.toDbDict()
cnt = qry.count()
#if this is a new record, insert it
if (cnt == 0):
mongoData.insert(tempRec)
#if this is an existing record, update it
elif (cnt == 1):
print 'foundone!!'
dbDict = qry.next()
temp = dbDict['_id']
dbDict = tempRec
dbDict['_id'] = temp
mongoData.save(dbDict)
else:
print 'strange, there is more than 1 DST record for', rec.time
del templist
def mapKpMongo(sYear, eYear=None):
"""This function reads kp data from the GFZ Potsdam FTP server via anonymous FTP connection and maps it to the mongodb.
.. warning::
In general, nobody except the database admins will need to use this function
**Args**:
* **sYear** (int): the year to begin mapping data
* [**eYear**] (int or None): the end year for mapping data. if this is None, eYear will be sYear. default=None
**Returns**:
* Nothing.
**Example**:
::
gme.ind.mapKpMongo(1985,eTime=1986)
written by AJ, 20130123
"""
import davitpy.pydarn.sdio.dbUtils as db
from davitpy import rcParams
import datetime as dt
if (eYear == None): eYear = sYear
assert (eYear >= sYear), 'error, end year greater than start year'
mongoData = db.getDataConn(username=rcParams['DBWRITEUSER'],password=rcParams['DBWRITEPASS'],\
dbAddress=rcParams['SDDB'],dbName='gme',collName='kp')
#set up all of the indices
mongoData.ensure_index('time')
mongoData.ensure_index('kp')
mongoData.ensure_index('ap')
mongoData.ensure_index('kpSum')
mongoData.ensure_index('apMean')
mongoData.ensure_index('sunspot')
#read the kp data from the FTP server
datalist = []
for yr in range(sYear, eYear + 1):
templist = readKpFtp(dt.datetime(yr, 1, 1), dt.datetime(yr + 1, 1, 1))
if (templist == None): continue
for rec in templist:
#check if a duplicate record exists
qry = mongoData.find({'time': rec.time})
tempRec = rec.toDbDict()
cnt = qry.count()
#if this is a new record, insert it
if (cnt == 0):
mongoData.insert(tempRec)
#if this is an existing record, update it
elif (cnt == 1):
print 'foundone!!'
dbDict = qry.next()
temp = dbDict['_id']
dbDict = tempRec
dbDict['_id'] = temp
mongoData.save(dbDict)
else:
print 'strange, there is more than 1 record for', rec.time
def mapDstMongo(sYear,eYear=None):
"""This function reads dst data from wdc and puts it in mongodb
.. warning::
In general, nobody except the database admins will need to use this function
**Args**:
* **sYear** (int): the year to begin mapping data
* [**eYear**] (int or None): the end year for mapping data. if this is None, eYear will be sYear
**Returns**:
* Nothing.
**Example**:
::
gme.ind.mapDstMongo(1997)
written by AJ, 20130123
"""
import davitpy.pydarn.sdio.dbUtils as db
from davitpy import rcParams
import datetime as dt
#check inputs
assert(isinstance(sYear,int)),'error, sYear must be int'
if(eYear == None): eYear=sYear
assert(isinstance(eYear,int)),'error, sYear must be None or int'
assert(eYear >= sYear), 'error, end year greater than start year'
#get data connection
mongoData = db.getDataConn(username=rcParams['DBWRITEUSER'],password=rcParams['DBWRITEPASS'],\
dbAddress=rcParams['SDDB'],dbName='gme',collName='dst')
#set up all of the indices
mongoData.ensure_index('time')
mongoData.ensure_index('dst')
for yr in range(sYear,eYear+1):
#1 year at a time, to not fill up RAM
templist = readDstWeb(dt.datetime(yr,1,1),dt.datetime(yr,12,31))
for rec in templist:
#check if a duplicate record exists
qry = mongoData.find({'time':rec.time})
print rec.time
tempRec = rec.toDbDict()
cnt = qry.count()
#if this is a new record, insert it
if(cnt == 0): mongoData.insert(tempRec)
#if this is an existing record, update it
elif(cnt == 1):
print 'foundone!!'
dbDict = qry.next()
temp = dbDict['_id']
dbDict = tempRec
dbDict['_id'] = temp
mongoData.save(dbDict)
else:
print 'strange, there is more than 1 DST record for',rec.time
del templist
def mapKpMongo(sYear,eYear=None):
"""This function reads kp data from the GFZ Potsdam FTP server via anonymous FTP connection and maps it to the mongodb.
.. warning::
In general, nobody except the database admins will need to use this function
**Args**:
* **sYear** (int): the year to begin mapping data
* [**eYear**] (int or None): the end year for mapping data. if this is None, eYear will be sYear. default=None
**Returns**:
* Nothing.
**Example**:
::
gme.ind.mapKpMongo(1985,eTime=1986)
written by AJ, 20130123
"""
import davitpy.pydarn.sdio.dbUtils as db
from davitpy import rcParams
import datetime as dt
if(eYear == None): eYear=sYear
assert(eYear >= sYear), 'error, end year greater than start year'
mongoData = db.getDataConn(username=rcParams['DBWRITEUSER'],password=rcParams['DBWRITEPASS'],\
dbAddress=rcParams['SDDB'],dbName='gme',collName='kp')
#set up all of the indices
mongoData.ensure_index('time')
mongoData.ensure_index('kp')
mongoData.ensure_index('ap')
mongoData.ensure_index('kpSum')
mongoData.ensure_index('apMean')
mongoData.ensure_index('sunspot')
#read the kp data from the FTP server
datalist = []
for yr in range(sYear,eYear+1):
templist = readKpFtp(dt.datetime(yr,1,1), dt.datetime(yr+1,1,1))
if(templist == None): continue
for rec in templist:
#check if a duplicate record exists
qry = mongoData.find({'time':rec.time})
tempRec = rec.toDbDict()
cnt = qry.count()
#if this is a new record, insert it
if(cnt == 0): mongoData.insert(tempRec)
#if this is an existing record, update it
elif(cnt == 1):
print 'foundone!!'
dbDict = qry.next()
temp = dbDict['_id']
dbDict = tempRec
dbDict['_id'] = temp
mongoData.save(dbDict)
else:
print 'strange, there is more than 1 record for',rec.time
def readDst(sTime=None,eTime=None,dst=None):
"""This function reads dst data from the mongodb.
**Args**:
* [**sTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the earliest time you want data for, default=None
* [**eTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the latest time you want data for. if this is None, end Time will be 1 day after sTime. default = None
* [**dst**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with dst values in the range [a,b] will be returned. default = None
**Returns**:
* **dstList** (list or None): if data is found, a list of :class:`gme.ind.dst.dstRec` objects matching the input parameters is returned. If no data is found, None is returned.
**Example**:
::
import datetime as dt
dstList = gme.ind.readDst(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),dst=[-50,50])
written by AJ, 20130131
"""
import datetime as dt
import davitpy.pydarn.sdio.dbUtils as db
#check all the inputs for validity
assert(sTime == None or isinstance(sTime,dt.datetime)), \
'error, sTime must be a datetime object'
assert(eTime == None or isinstance(eTime,dt.datetime)), \
'error, eTime must be either None or a datetime object'
assert(dst == None or (isinstance(dst,list) and \
isinstance(dst[0],(int,float)) and isinstance(dst[1],(int,float)))), \
'error,dst must None or a list of 2 numbers'
if(eTime == None and sTime != None): eTime = sTime+dt.timedelta(days=1)
qryList = []
#if arguments are provided, query for those
if(sTime != None): qryList.append({'time':{'$gte':sTime}})
if(eTime != None): qryList.append({'time':{'$lte':eTime}})
if(dst != None):
qryList.append({'dst':{'$gte':min(dst)}})
qryList.append({'dst':{'$lte':max(dst)}})
#construct the final query definition
qryDict = {'$and': qryList}
#connect to the database
dstData = db.getDataConn(dbName='gme',collName='dst')
#do the query
if(qryList != []): qry = dstData.find(qryDict)
else: qry = dstData.find()
if(qry.count() > 0):
dstList = []
for rec in qry.sort('time'):
dstList.append(dstRec(dbDict=rec))
print '\nreturning a list with',len(dstList),'records of dst data'
return dstList
#if we didn't find anything on the mongodb
else:
print '\ncould not find requested data in the mongodb'
return None
def readDst(sTime=None, eTime=None, dst=None):
"""This function reads dst data from the mongodb.
**Args**:
* [**sTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the earliest time you want data for, default=None
* [**eTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the latest time you want data for. if this is None, end Time will be 1 day after sTime. default = None
* [**dst**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with dst values in the range [a,b] will be returned. default = None
**Returns**:
* **dstList** (list or None): if data is found, a list of :class:`gme.ind.dst.dstRec` objects matching the input parameters is returned. If no data is found, None is returned.
**Example**:
::
import datetime as dt
dstList = gme.ind.readDst(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),dst=[-50,50])
written by AJ, 20130131
"""
import datetime as dt
import davitpy.pydarn.sdio.dbUtils as db
#check all the inputs for validity
assert(sTime == None or isinstance(sTime,dt.datetime)), \
'error, sTime must be a datetime object'
assert(eTime == None or isinstance(eTime,dt.datetime)), \
'error, eTime must be either None or a datetime object'
assert(dst == None or (isinstance(dst,list) and \
isinstance(dst[0],(int,float)) and isinstance(dst[1],(int,float)))), \
'error,dst must None or a list of 2 numbers'
if (eTime == None and sTime != None): eTime = sTime + dt.timedelta(days=1)
qryList = []
#if arguments are provided, query for those
if (sTime != None): qryList.append({'time': {'$gte': sTime}})
if (eTime != None): qryList.append({'time': {'$lte': eTime}})
if (dst != None):
qryList.append({'dst': {'$gte': min(dst)}})
qryList.append({'dst': {'$lte': max(dst)}})
#construct the final query definition
qryDict = {'$and': qryList}
#connect to the database
dstData = db.getDataConn(dbName='gme', collName='dst')
#do the query
if (qryList != []): qry = dstData.find(qryDict)
else: qry = dstData.find()
if (qry.count() > 0):
dstList = []
for rec in qry.sort('time'):
dstList.append(dstRec(dbDict=rec))
print '\nreturning a list with', len(dstList), 'records of dst data'
return dstList
#if we didn't find anything on the mongodb
else:
print '\ncould not find requested data in the mongodb'
return None
def mapOmniMongo(sYear,eYear=None,res=5):
"""This function reads omni data from the NASA SPDF FTP server via anonymous FTP connection and maps it to the mongodb.
.. warning::
In general, nobody except the database admins will need to use this function
**Args**:
* **sYear** (int): the year to begin mapping data
* [**eYear**] (int or None): the end year for mapping data. if this is None, eYear will be sYear
* [**res**] (int): the time resolution for mapping data. Can be either 1 or 5. default=5
**Returns**:
* Nothing.
**Example**:
::
gme.ind.mapOmniMongo(1997,res=1)
written by AJ, 20130123
"""
import davitpy.pydarn.sdio.dbUtils as db
from davitpy import rcParams
import datetime as dt
#check inputs
assert(res == 1 or res == 5),'error, res must be either 1 or 5'
assert(isinstance(sYear,int)),'error, sYear must be int'
if(eYear == None): eYear=sYear
assert(isinstance(eYear,int)),'error, sYear must be None or int'
assert(eYear >= sYear), 'error, end year greater than start year'
#get data connection
mongoData = db.getDataConn(username=rcParams['DBWRITEUSER'],password=rcParams['DBWRITEPASS'],\
dbAddress=rcParams['SDDB'],dbName='gme',collName='omni')
#set up all of the indices
mongoData.ensure_index('time')
mongoData.ensure_index('res')
mongoData.ensure_index('bx')
mongoData.ensure_index('bye')
mongoData.ensure_index('bze')
mongoData.ensure_index('bym')
mongoData.ensure_index('bzm')
mongoData.ensure_index('pDyn')
mongoData.ensure_index('ae')
mongoData.ensure_index('symh')
#read the omni data from the FTP server
for yr in range(sYear,eYear+1):
for mon in range(1,13):
templist = readOmniFtp(dt.datetime(yr,mon,1),dt.datetime(yr,mon,1)+dt.timedelta(days=31),res=res)
if(templist == None): continue
for rec in templist:
#check if a duplicate record exists
qry = mongoData.find({'$and':[{'time':rec.time},{'res':rec.res}]})
print rec.time
tempRec = rec.toDbDict()
cnt = qry.count()
#if this is a new record, insert it
if(cnt == 0): mongoData.insert(tempRec)
#if this is an existing record, update it
elif(cnt == 1):
print 'foundone!!'
dbDict = qry.next()
temp = dbDict['_id']
dbDict = tempRec
dbDict['_id'] = temp
mongoData.save(dbDict)
else:
print 'strange, there is more than 1 record for',rec.time
def readOmni(sTime,eTime=None,res=5,bx=None,bye=None,bze=None,bym=None,bzm=None,pDyn=None,ae=None,symh=None):
"""This function reads omni data. First, it will try to get it from the mongodb, and if it can't find it, it will look on the NASA SPDF FTP server using :func:`readOmniFtp`
**Args**:
* **sTime** (`datetime <http://tinyurl.com/bl352yx>`_ or None): the earliest time you want data for
* [**eTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the latest time you want data for. if this is None, end Time will be 1 day after sTime. default = None
* [**res**] (int): the time reolution of data desired. This can be either 1 or 5. default = 5
* [**bx**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bx values in the range [a,b] will be returned. default = None
* [**bx**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bx values in the range [a,b] will be returned. default = None
* [**bye**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bye values in the range [a,b] will be returned. default = None
* [**bze**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bze values in the range [a,b] will be returned. default = None
* [**bym**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bym values in the range [a,b] will be returned. default = None
* [**bzm**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bzm values in the range [a,b] will be returned. default = None
* [**pDyn**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with pDyn values in the range [a,b] will be returned. default = None
* [**ae**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with ae values in the range [a,b] will be returned. default = None
* [**symh**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with symh values in the range [a,b] will be returned. default = None
**Returns**:
* **omniList** (list or None): if data is found, a list of :class:`omniRec` objects matching the input parameters is returned. If no data is found, None is returned.
**Example**:
::
import datetime as dt
omniList = gme.ind.readOmni(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),bx=[0,5.5],bye=[-1,3.5],bze=[-10,0],ae=[0,56.3])
written by AJ, 20130128
"""
import datetime as dt
import davitpy.pydarn.sdio.dbUtils as db
#check all the inputs for validity
assert(isinstance(sTime,dt.datetime)), \
'error, sTime must be a datetime object'
assert(eTime == None or isinstance(eTime,dt.datetime)), \
'error, eTime must be either None or a datetime object'
assert(res==5 or res == 1), 'error, res must be either 1 or 5'
var = locals()
for name in ['bx','bye','bze','bym','bzm','pDyn','ae','symh']:
assert(var[name] == None or (isinstance(var[name],list) and \
isinstance(var[name][0],(int,float)) and isinstance(var[name][1],(int,float)))), \
'error,'+name+' must None or a list of 2 numbers'
if(eTime == None): eTime = sTime+dt.timedelta(days=1)
qryList = []
#if arguments are provided, query for those
qryList.append({'time':{'$gte':sTime}})
qryList.append({'res':res})
if(eTime != None): qryList.append({'time':{'$lte':eTime}})
var = locals()
for name in ['bx','bye','bze','bym','bzm','pDyn','ae','symh']:
if(var[name] != None):
qryList.append({name:{'$gte':min(var[name])}})
qryList.append({name:{'$lte':max(var[name])}})
#construct the final query definition
qryDict = {'$and': qryList}
#connect to the database
omniData = db.getDataConn(dbName='gme',collName='omni')
#do the query
if(qryList != []): qry = omniData.find(qryDict)
else: qry = omniData.find()
if(qry.count() > 0):
omniList = []
for rec in qry.sort('time'):
omniList.append(omniRec(dbDict=rec))
print '\nreturning a list with',len(omniList),'records of omni data'
return omniList
#if we didn't find anything on the mongodb
else:
print '\ncould not find requested data in the mongodb'
print 'we will look on the ftp server, but your conditions will be (mostly) ignored'
#read from ftp server
omniList = readOmniFtp(sTime, eTime)
if(omniList != None):
print '\nreturning a list with',len(omniList),'recs of omni data'
return omniList
else:
print '\n no data found on FTP server, returning None...'
return None
def readPoes(stime, eTime=None, satnum=None, folat=None, folon=None, ted=None,
echar=None, pchar=None):
"""This function reads POES data.
Parameters
----------
stime : (datetime or None)
the earliest time you want data for
eTime : (datetime or None)
the latest time you want data for. if this is None, end Time will be 1
day after stime. (default=None)
satnum : (int)
the satellite you want data for. eg 17 for noaa17. if this is None,
data for all satellites will be returned. (default=None)
folat : (list or None)
if this is not None, it must be a 2-element list of numbers, [a,b]. In
this case, only data with bx values in the range [a,b] will be returned.
(default=None)
folon : (list or None)
if this is not None, it must be a 2-element list of numbers, [a,b]. In
this case, only data with bye values in the range [a,b] will be returned
(default=None)
ted : (list or None)
if this is not None, it must be a 2-element list of numbers, [a,b]. In
this case, only data with bze values in the range [a,b] will be returned
(default=None)
echar : (list or None)
if this is not None, it must be a 2-element list of numbers, [a,b]. In
this case, only data with bym values in the range [a,b] will be returned
(default=None)
pchar : (list or None)
if this is not None, it must be a 2-element list of numbers, [a,b]. In
this case, only data with bzm values in the range [a,b] will be returned
(default=None)
Returns
-------
poesList : (list or None)
if data is found, a list of class poesRec objects matching the input
parameters is returned. If no data is found, None is returned.
Notes
--------
First, it will try to get it from the mongodb, and if it can't find it, it
will look on the NOAA NGDC FTP server using :func:`readPoesFtp`. The data
are 16-second averages
Example
-------
import datetime as dt
poesList = gme.sat.readPoes(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),folat=[60,80])
written by AJ, 20130131
"""
import datetime as dt
import davitpy.pydarn.sdio.dbUtils as db
# check all the inputs for validity
assert isinstance(stime, dt.datetime), \
logging.error('stime must be a datetime object')
assert eTime == None or isinstance(eTime,dt.datetime), \
logging.error('eTime must be either None or a datetime object')
assert satnum == None or isinstance(satnum,int), \
logging.error('satnum must be an int')
var = locals()
for name in ['folat','folon','ted','echar','pchar']:
assert(var[name] == None or (isinstance(var[name],list) and
isinstance(var[name][0],(int,float)) and
isinstance(var[name][1],(int,float)))), \
logging.error(name + ' must None or a list of 2 numbers')
if(eTime == None):
eTime = stime + dt.timedelta(days=1)
qryList = []
# if arguments are provided, query for those
qryList.append({'time':{'$gte':stime}})
if(eTime != None):
qryList.append({'time':{'$lte':eTime}})
if(satnum != None):
qryList.append({'satnum':satnum})
var = locals()
for name in ['folat','folon','ted','echar','pchar']:
if(var[name] != None):
qryList.append({name:{'$gte':min(var[name])}})
qryList.append({name:{'$lte':max(var[name])}})
# construct the final query definition
qryDict = {'$and': qryList}
# connect to the database
poesData = db.getDataConn(dbName='gme', collName='poes')
# do the query
qry = poesData.find(qryDict) if qryList != [] else poesData.find()
if(qry.count() > 0):
poesList = []
for rec in qry.sort('time'):
poesList.append(poesRec(dbDict=rec))
logging.info('nreturning a list with '+ len(poesList) +
' records of poes data')
return poesList
# if we didn't find anything on the mongodb
else:
logging.info('could not find requested data in the mongodb')
return None
def readAe(sTime=None, eTime=None, res=60, ae=None, al=None, au=None, ao=None):
"""This function reads ae data from the mongodb. **The data are 1-minute values**
Parameters
----------
sTime : Optional[datetime]
the earliest time you want data for, default=None
eTime : Optional[datetime]
the latest time you want data for. if this is None, end Time will be
1 day after sTime. default = None
res : Optional[int]
the time resolution desired in minutes. Valid inputs are 1 and 60.
default = 60
ae : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with ae values in the range [a,b] will be
returned. default = None
al : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with al values in the range [a,b] will be
returned. default = None
au : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with au values in the range [a,b] will be
returned. default = None
ao : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with ao values in the range [a,b] will be
returned. default = None
Returns
-------
aeList : list
if data is found, a list of class gme.ind.ae.aeRec objects matching
the input parameters is returned. If no data is found, None is
returned.
Example
-------
import datetime as dt
aeList = gme.ind.readAe(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),res=60,ao=[-50,50])
written by AJ, 20130131
"""
import datetime as dt
import davitpy.pydarn.sdio.dbUtils as db
#check all the inputs for validity
assert(sTime == None or isinstance(sTime,dt.datetime)), \
logging.error('sTime must be a datetime object')
assert(eTime == None or isinstance(eTime,dt.datetime)), \
logging.error('eTime must be either None or a datetime object')
assert (res == 60 or res == 1), logging.error('res must be 1 or 60')
var = locals()
for name in ['ae', 'al', 'au', 'ao']:
assert(var[name] == None or (isinstance(var[name],list) and \
isinstance(var[name][0],(int,float)) and isinstance(var[name][1],(int,float)))), \
logging.error(name + ' must None or a list of 2 numbers')
if (eTime == None and sTime != None): eTime = sTime + dt.timedelta(days=1)
qryList = []
#if arguments are provided, query for those
if (sTime != None): qryList.append({'time': {'$gte': sTime}})
if (eTime != None): qryList.append({'time': {'$lte': eTime}})
qryList.append({'res': res})
var = locals()
for name in ['ae', 'al', 'au', 'ao']:
if (var[name] != None):
qryList.append({name: {'$gte': min(var[name])}})
qryList.append({name: {'$lte': max(var[name])}})
#construct the final query definition
qryDict = {'$and': qryList}
#connect to the database
aeData = db.getDataConn(dbName='gme', collName='ae')
#do the query
if (qryList != []): qry = aeData.find(qryDict)
else: qry = aeData.find()
if (qry.count() > 0):
aeList = []
for rec in qry.sort('time'):
aeList.append(aeRec(dbDict=rec))
logging.info('\nreturning a list with ' + len(aeList) +
'records of ae data')
return aeList
#if we didn't find anything on the mongodb
else:
logging.info('\ncould not find requested data in the mongodb')
return None
def mapOmniMongo(sYear, eYear=None, res=5):
"""This function reads omni data from the NASA SPDF FTP server via anonymous FTP connection and maps it to the mongodb.
.. warning::
In general, nobody except the database admins will need to use this function
**Args**:
* **sYear** (int): the year to begin mapping data
* [**eYear**] (int or None): the end year for mapping data. if this is None, eYear will be sYear
* [**res**] (int): the time resolution for mapping data. Can be either 1 or 5. default=5
**Returns**:
* Nothing.
**Example**:
::
gme.ind.mapOmniMongo(1997,res=1)
written by AJ, 20130123
"""
import davitpy.pydarn.sdio.dbUtils as db
from davitpy import rcParams
import datetime as dt
#check inputs
assert (res == 1 or res == 5), 'error, res must be either 1 or 5'
assert (isinstance(sYear, int)), 'error, sYear must be int'
if (eYear == None): eYear = sYear
assert (isinstance(eYear, int)), 'error, sYear must be None or int'
assert (eYear >= sYear), 'error, end year greater than start year'
#get data connection
mongoData = db.getDataConn(username=rcParams['DBWRITEUSER'],password=rcParams['DBWRITEPASS'],\
dbAddress=rcParams['SDDB'],dbName='gme',collName='omni')
#set up all of the indices
mongoData.ensure_index('time')
mongoData.ensure_index('res')
mongoData.ensure_index('bx')
mongoData.ensure_index('bye')
mongoData.ensure_index('bze')
mongoData.ensure_index('bym')
mongoData.ensure_index('bzm')
mongoData.ensure_index('pDyn')
mongoData.ensure_index('ae')
mongoData.ensure_index('symh')
#read the omni data from the FTP server
for yr in range(sYear, eYear + 1):
for mon in range(1, 13):
templist = readOmniFtp(dt.datetime(yr, mon, 1),
dt.datetime(yr, mon, 1) +
dt.timedelta(days=31),
res=res)
if (templist == None): continue
for rec in templist:
#check if a duplicate record exists
qry = mongoData.find(
{'$and': [{
'time': rec.time
}, {
'res': rec.res
}]})
print rec.time
tempRec = rec.toDbDict()
cnt = qry.count()
#if this is a new record, insert it
if (cnt == 0):
mongoData.insert(tempRec)
#if this is an existing record, update it
elif (cnt == 1):
print 'foundone!!'
dbDict = qry.next()
temp = dbDict['_id']
dbDict = tempRec
dbDict['_id'] = temp
mongoData.save(dbDict)
else:
print 'strange, there is more than 1 record for', rec.time
def readKp(sTime=None,eTime=None,kpMin=None,apMin=None,kpSum=None,apMean=None,sunspot=None):
"""This function reads kp data. First, it will try to get it from the mongodb,
and if it can't find it, it will look on the GFZ ftp server using
gme.ind.kp.readKpFtp
Parameters
----------
sTime : Optional[datetime]
the earliest time you want data for. if this is None, start time will
be the earliest record found. default=None
eTime : Optional[datetime]
the latest time you want data for. if this is None, end Time will be
latest record found. default=None
kpMin : Optional[int]
specify this to only return data from dates with a 3-hour kp value of
minimum kpMin. if this is none, it will be ignored. default=None
apMin : Optional[int]
specify this to only return data from dates with a 3-hour ap value of
minimum apMin. if this is none, it will be ignored. default=None
kpSum : Optional[list]
this must be a 2 element list of integers. if this is specified,
only dates with kpSum values in the range [a,b] will be returned. if
this is None, it will be ignored. default=None
apMean : Optional[list]
this must be a 2 element list of integers. if this is specified, only
dates with apMean values in the range [a,b] will be returned. if this
is None, it will be ignored. default=None
sunspot : Optionl[list]
this must be a 2 element list of integers. if this is specified, only
dates with sunspot values in the range [a,b] will be returned. if
this is None, it will be ignored. default=None
Returns
-------
kpList : list
if data is found, a list of class gme.ind.kp.kpDay objects matching
the input parameters is returned. If not data is found, None is
returned.
Example
-------
import datetime as dt
kpList = gme.ind.readKp(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),kpMin=2,apMin=1,kpSum=[0,10],apMean=[0,50],sunspot=[6,100])
written by AJ, 20130123
"""
import datetime as dt
import davitpy.pydarn.sdio.dbUtils as db
#check all the inputs for validity
assert(sTime == None or isinstance(sTime,dt.datetime)), \
logging.error('sTime must be either None or a datetime object')
assert(eTime == None or isinstance(eTime,dt.datetime)), \
logging.error('eTime must be either None or a datetime object')
assert(kpMin == None or isinstance(kpMin,int)), \
logging.error('kpMin must be either None or an int')
assert(apMin == None or isinstance(apMin,int)), \
logging.error('apMin must be either None or an int')
assert(kpSum == None or (isinstance(kpSum,list) and len(kpSum) == 2 and \
isinstance(kpSum[0], int) and isinstance(kpSum[1], int))), \
logging.error('kpSum must be either None or a 2 element list')
assert(apMean == None or (isinstance(apMean,list) and len(apMean) == 2and \
isinstance(apMean[0], int) and isinstance(apMean[1], int))), \
logging.error('apMean must be either None or a 2 element list')
assert(sunspot == None or (isinstance(sunspot,list) and len(sunspot) == 2and \
isinstance(sunspot[0], int) and isinstance(sunspot[1], int))), \
logging.error('sunspot must be either None or a 2 element list')
qryList = []
#if arguments are provided, query for those
if(sTime != None): qryList.append({'time':{'$gte':sTime}})
if(eTime != None): qryList.append({'time':{'$lte':eTime}})
if(kpMin != None): qryList.append({'kp':{'$gte':kpMin}})
if(apMin != None): qryList.append({'ap':{'$gte':kpMin}})
if(kpSum != None): qryList.append({'kpSum':{'$gte':min(kpSum)}})
if(kpSum != None): qryList.append({'kpSum':{'$lte':max(kpSum)}})
if(apMean != None): qryList.append({'apMean':{'$gte':min(apMean)}})
if(apMean != None): qryList.append({'apMean':{'$lte':max(apMean)}})
if(sunspot != None): qryList.append({'sunspot':{'$gte':min(sunspot)}})
if(sunspot != None): qryList.append({'sunspot':{'$lte':max(sunspot)}})
#construct the final query definition
qryDict = {'$and': qryList}
#connect to the database
kpData = db.getDataConn(dbName='gme',collName='kp')
#do the query
if(qryList != []): qry = kpData.find(qryDict)
else: qry = kpData.find()
if(qry.count() > 0):
kpList = []
for rec in qry.sort('time'):
kpList.append(kpDay(dbDict=rec))
logging.info('\nreturning a list with',len(kpList),'days of kp data')
return kpList
#if we didn't find anything ont he mongodb
else:
logging.warning('\ncould not find requested data in the mongodb')
logging.warning('we will look on the ftp server, but your conditions will be (mostly) ignored')
if(sTime == None):
logging.info('start time for search set to 1980...')
sTime = dt.datetime(1980,1,1)
kpList = []
if(eTime == None): eTime = dt.now()
for yr in range(sTime.year,eTime.year+1):
tmpList = readKpFtp(dt.datetime(yr,1,1), eTime=dt.datetime(yr,12,31))
if(tmpList == None): continue
for x in tmpList:
kpList.append(x)
if(kpList != []):
logging.info('\nreturning a list with ' + len(kpList) + ' days of kp data')
return kpList
else:
logging.info('\n no data found on FTP server, returning None...')
return None
def mapSymAsyMongo(sYear,eYear=None):
"""This function reads sym/asy data from wdc and puts it in mongodb
.. warning::
In general, nobody except the database admins will need to
use this function
Parameters
----------
sYear : int
the year to begin mapping data
eYear : Optional[int]
the end year for mapping data. if this is None, eYear
will be sYear
Returns
-------
Nothing
Example
-------
gme.ind.mapSymAsyMongo(2001)
written by AJ, 20130123
"""
import davitpy.pydarn.sdio.dbUtils as db
from davitpy import rcParams
import datetime as dt
#check inputs
assert(isinstance(sYear,int)), logging.error('sYear must be int')
if(eYear == None): eYear=sYear
assert(isinstance(eYear,int)), logging.error('sYear must be None or int')
assert(eYear >= sYear), logging.error('end year greater than start year')
#get data connection
mongoData = db.getDataConn(username=rcParams['DBWRITEUSER'],password=rcParams['DBWRITEPASS'],\
dbAddress=rcParams['SDDB'],dbName='gme',collName='symasy')
#set up all of the indices
mongoData.ensure_index('time')
mongoData.ensure_index('symh')
mongoData.ensure_index('symd')
mongoData.ensure_index('asyh')
mongoData.ensure_index('asyd')
for yr in range(sYear,eYear+1):
#1 day at a time, to not fill up RAM
templist = readSymAsyWeb(dt.datetime(yr,1,1),dt.datetime(yr,1,1)+dt.timedelta(days=366))
if(templist == None): continue
for rec in templist:
#check if a duplicate record exists
qry = mongoData.find({'time':rec.time})
logging.debug(rec.time)
tempRec = rec.toDbDict()
cnt = qry.count()
#if this is a new record, insert it
if(cnt == 0): mongoData.insert(tempRec)
#if this is an existing record, update it
elif(cnt == 1):
logging.debug('found one!!')
dbDict = qry.next()
temp = dbDict['_id']
dbDict = tempRec
dbDict['_id'] = temp
mongoData.save(dbDict)
else:
logging.warning('strange, there is more than 1 Sym/Asy record for' + rec.time)
del templist
def readSymAsy(sTime=None,eTime=None,symh=None,symd=None,asyh=None,asyd=None):
"""This function reads sym/asy data from the mongodb.
Parameters
----------
sTime : Optional[datetime]
the earliest time you want data for, default=None
eTime : Optional[datetime]
the latest time you want data for. if this is None, end Time will
be 1 day after sTime. default = None
symh : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with symh values in the range [a,b] will be
returned. default = None
symd : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with symd values in the range [a,b] will be
returned. default = None
asyh : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with asyh values in the range [a,b] will be
returned. default = None
asyd : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with asyd values in the range [a,b] will be
returned. default = None
Returns
-------
symList : list
if data is found, a list of class gme.ind.symasy.symAsyRec objects
matching the input parameters is returned. If no data is found,
None is returned.
Example
-------
import datetime as dt
symList = gme.ind.readSymAsy(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),symh=[5,50],asyd=[-10,0])
written by AJ, 20130131
"""
import datetime as dt
import davitpy.pydarn.sdio.dbUtils as db
#check all the inputs for validity
assert(sTime == None or isinstance(sTime,dt.datetime)), \
logging.error('sTime must be a datetime object')
assert(eTime == None or isinstance(eTime,dt.datetime)), \
logging.error('eTime must be either None or a datetime object')
var = locals()
for name in ['symh','symd','asyh','asyd']:
assert(var[name] == None or (isinstance(var[name],list) and \
isinstance(var[name][0],(int,float)) and isinstance(var[name][1],(int,float)))), \
logging.error(name + ' must None or a list of 2 numbers')
if(eTime == None and sTime != None): eTime = sTime+dt.timedelta(days=1)
qryList = []
#if arguments are provided, query for those
if(sTime != None): qryList.append({'time':{'$gte':sTime}})
if(eTime != None): qryList.append({'time':{'$lte':eTime}})
var = locals()
for name in ['symh','symd','asyh','asyd']:
if(var[name] != None):
qryList.append({name:{'$gte':min(var[name])}})
qryList.append({name:{'$lte':max(var[name])}})
#construct the final query definition
qryDict = {'$and': qryList}
#connect to the database
symData = db.getDataConn(dbName='gme',collName='symasy')
#do the query
if(qryList != []): qry = symData.find(qryDict)
else: qry = symData.find()
if(qry.count() > 0):
symList = []
for rec in qry.sort('time'):
symList.append(symAsyRec(dbDict=rec))
logging.info('\nreturning a list with ' + str(len(symList)) + ' records of sym/asy data')
return symList
#if we didn't find anything on the mongodb
else:
logging.info('\ncould not find requested data in the mongodb')
return None
def readKp(sTime=None,
eTime=None,
kpMin=None,
apMin=None,
kpSum=None,
apMean=None,
sunspot=None):
"""This function reads kp data. First, it will try to get it from the mongodb,
and if it can't find it, it will look on the GFZ ftp server using
gme.ind.kp.readKpFtp
Parameters
----------
sTime : Optional[datetime]
the earliest time you want data for. if this is None, start time will
be the earliest record found. default=None
eTime : Optional[datetime]
the latest time you want data for. if this is None, end Time will be
latest record found. default=None
kpMin : Optional[int]
specify this to only return data from dates with a 3-hour kp value of
minimum kpMin. if this is none, it will be ignored. default=None
apMin : Optional[int]
specify this to only return data from dates with a 3-hour ap value of
minimum apMin. if this is none, it will be ignored. default=None
kpSum : Optional[list]
this must be a 2 element list of integers. if this is specified,
only dates with kpSum values in the range [a,b] will be returned. if
this is None, it will be ignored. default=None
apMean : Optional[list]
this must be a 2 element list of integers. if this is specified, only
dates with apMean values in the range [a,b] will be returned. if this
is None, it will be ignored. default=None
sunspot : Optionl[list]
this must be a 2 element list of integers. if this is specified, only
dates with sunspot values in the range [a,b] will be returned. if
this is None, it will be ignored. default=None
Returns
-------
kpList : list
if data is found, a list of class gme.ind.kp.kpDay objects matching
the input parameters is returned. If not data is found, None is
returned.
Example
-------
import datetime as dt
kpList = gme.ind.readKp(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),kpMin=2,apMin=1,kpSum=[0,10],apMean=[0,50],sunspot=[6,100])
written by AJ, 20130123
"""
import datetime as dt
import davitpy.pydarn.sdio.dbUtils as db
#check all the inputs for validity
assert(sTime == None or isinstance(sTime,dt.datetime)), \
logging.error('sTime must be either None or a datetime object')
assert(eTime == None or isinstance(eTime,dt.datetime)), \
logging.error('eTime must be either None or a datetime object')
assert(kpMin == None or isinstance(kpMin,int)), \
logging.error('kpMin must be either None or an int')
assert(apMin == None or isinstance(apMin,int)), \
logging.error('apMin must be either None or an int')
assert(kpSum == None or (isinstance(kpSum,list) and len(kpSum) == 2 and \
isinstance(kpSum[0], int) and isinstance(kpSum[1], int))), \
logging.error('kpSum must be either None or a 2 element list')
assert(apMean == None or (isinstance(apMean,list) and len(apMean) == 2and \
isinstance(apMean[0], int) and isinstance(apMean[1], int))), \
logging.error('apMean must be either None or a 2 element list')
assert(sunspot == None or (isinstance(sunspot,list) and len(sunspot) == 2and \
isinstance(sunspot[0], int) and isinstance(sunspot[1], int))), \
logging.error('sunspot must be either None or a 2 element list')
qryList = []
#if arguments are provided, query for those
if (sTime != None): qryList.append({'time': {'$gte': sTime}})
if (eTime != None): qryList.append({'time': {'$lte': eTime}})
if (kpMin != None): qryList.append({'kp': {'$gte': kpMin}})
if (apMin != None): qryList.append({'ap': {'$gte': kpMin}})
if (kpSum != None): qryList.append({'kpSum': {'$gte': min(kpSum)}})
if (kpSum != None): qryList.append({'kpSum': {'$lte': max(kpSum)}})
if (apMean != None): qryList.append({'apMean': {'$gte': min(apMean)}})
if (apMean != None): qryList.append({'apMean': {'$lte': max(apMean)}})
if (sunspot != None): qryList.append({'sunspot': {'$gte': min(sunspot)}})
if (sunspot != None): qryList.append({'sunspot': {'$lte': max(sunspot)}})
#construct the final query definition
qryDict = {'$and': qryList}
#connect to the database
kpData = db.getDataConn(dbName='gme', collName='kp')
#do the query
if (qryList != []): qry = kpData.find(qryDict)
else: qry = kpData.find()
if (qry.count() > 0):
kpList = []
for rec in qry.sort('time'):
kpList.append(kpDay(dbDict=rec))
logging.info('\nreturning a list with', len(kpList), 'days of kp data')
return kpList
#if we didn't find anything ont he mongodb
else:
logging.warning('\ncould not find requested data in the mongodb')
logging.warning(
'we will look on the ftp server, but your conditions will be (mostly) ignored'
)
if (sTime == None):
logging.info('start time for search set to 1980...')
sTime = dt.datetime(1980, 1, 1)
kpList = []
if (eTime == None): eTime = dt.now()
for yr in range(sTime.year, eTime.year + 1):
tmpList = readKpFtp(dt.datetime(yr, 1, 1),
eTime=dt.datetime(yr, 12, 31))
if (tmpList == None): continue
for x in tmpList:
kpList.append(x)
if (kpList != []):
logging.info('\nreturning a list with ' + str(len(kpList)) +
' days of kp data')
return kpList
else:
logging.info('\n no data found on FTP server, returning None...')
return None
def readSymAsy(sTime=None,
eTime=None,
symh=None,
symd=None,
asyh=None,
asyd=None):
"""This function reads sym/asy data from the mongodb.
Parameters
----------
sTime : Optional[datetime]
the earliest time you want data for, default=None
eTime : Optional[datetime]
the latest time you want data for. if this is None, end Time will
be 1 day after sTime. default = None
symh : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with symh values in the range [a,b] will be
returned. default = None
symd : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with symd values in the range [a,b] will be
returned. default = None
asyh : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with asyh values in the range [a,b] will be
returned. default = None
asyd : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with asyd values in the range [a,b] will be
returned. default = None
Returns
-------
symList : list
if data is found, a list of class gme.ind.symasy.symAsyRec objects
matching the input parameters is returned. If no data is found,
None is returned.
Example
-------
import datetime as dt
symList = gme.ind.readSymAsy(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),symh=[5,50],asyd=[-10,0])
written by AJ, 20130131
"""
import datetime as dt
import davitpy.pydarn.sdio.dbUtils as db
#check all the inputs for validity
assert(sTime == None or isinstance(sTime,dt.datetime)), \
logging.error('sTime must be a datetime object')
assert(eTime == None or isinstance(eTime,dt.datetime)), \
logging.error('eTime must be either None or a datetime object')
var = locals()
for name in ['symh', 'symd', 'asyh', 'asyd']:
assert(var[name] == None or (isinstance(var[name],list) and \
isinstance(var[name][0],(int,float)) and isinstance(var[name][1],(int,float)))), \
logging.error(name + ' must None or a list of 2 numbers')
if (eTime == None and sTime != None): eTime = sTime + dt.timedelta(days=1)
qryList = []
#if arguments are provided, query for those
if (sTime != None): qryList.append({'time': {'$gte': sTime}})
if (eTime != None): qryList.append({'time': {'$lte': eTime}})
var = locals()
for name in ['symh', 'symd', 'asyh', 'asyd']:
if (var[name] != None):
qryList.append({name: {'$gte': min(var[name])}})
qryList.append({name: {'$lte': max(var[name])}})
#construct the final query definition
qryDict = {'$and': qryList}
#connect to the database
symData = db.getDataConn(dbName='gme', collName='symasy')
#do the query
if (qryList != []): qry = symData.find(qryDict)
else: qry = symData.find()
if (qry.count() > 0):
symList = []
for rec in qry.sort('time'):
symList.append(symAsyRec(dbDict=rec))
logging.info('\nreturning a list with ' + str(len(symList)) +
' records of sym/asy data')
return symList
#if we didn't find anything on the mongodb
else:
logging.info('\ncould not find requested data in the mongodb')
return None
def readOmni(sTime,
eTime=None,
res=5,
bx=None,
bye=None,
bze=None,
bym=None,
bzm=None,
pDyn=None,
ae=None,
symh=None):
"""This function reads omni data. First, it will try to get it from the
mongodb, and if it can't find it, it will look on the NASA SPDF FTP
server using readOmniFtp
Parameters
----------
sTime : datetime
the earliest time you want data for
eTime : Optional[datetime]
the latest time you want data for. if this is None, end Time will
be 1 day after sTime. default = None
res : Optional[int]
the time reolution of data desired. This can be either 1
or 5. default = 5
bx : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with bx values in the range [a,b] will be
returned. default = None
bx : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with bx values in the range [a,b] will be
returned. default = None
bye : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with bye values in the range [a,b] will be
returned. default = None
bze : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with bze values in the range [a,b] will be
returned. default = None
bym : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with bym values in the range [a,b] will be
returned. default = None
bzm : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with bzm values in the range [a,b] will be
returned. default = None
pDyn : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with pDyn values in the range [a,b] will be
returned. default = None
ae : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with ae values in the range [a,b] will be
returned. default = None
symh : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with symh values in the range [a,b] will be
returned. default = None
Returns
-------
omniList : list
if data is found, a list of class omniRec objects matching the
input parameters is returned. If no data is found, None is
returned.
Example
-------
import datetime as dt
omniList = gme.ind.readOmni(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),bx=[0,5.5],bye=[-1,3.5],bze=[-10,0],ae=[0,56.3])
written by AJ, 20130128
"""
import datetime as dt
import davitpy.pydarn.sdio.dbUtils as db
#check all the inputs for validity
assert(isinstance(sTime,dt.datetime)), \
logging.error('sTime must be a datetime object')
assert(eTime == None or isinstance(eTime,dt.datetime)), \
logging.error('eTime must be either None or a datetime object')
assert (res == 5 or res == 1), logging.error('res must be either 1 or 5')
var = locals()
for name in ['bx', 'bye', 'bze', 'bym', 'bzm', 'pDyn', 'ae', 'symh']:
assert(var[name] == None or (isinstance(var[name],list) and \
isinstance(var[name][0],(int,float)) and isinstance(var[name][1],(int,float)))), \
logging.error(name + ' must None or a list of 2 numbers')
if (eTime == None): eTime = sTime + dt.timedelta(days=1)
qryList = []
#if arguments are provided, query for those
qryList.append({'time': {'$gte': sTime}})
qryList.append({'res': res})
if (eTime != None): qryList.append({'time': {'$lte': eTime}})
var = locals()
for name in ['bx', 'bye', 'bze', 'bym', 'bzm', 'pDyn', 'ae', 'symh']:
if (var[name] != None):
qryList.append({name: {'$gte': min(var[name])}})
qryList.append({name: {'$lte': max(var[name])}})
#construct the final query definition
qryDict = {'$and': qryList}
#connect to the database
omniData = db.getDataConn(dbName='gme', collName='omni')
#do the query
if (qryList != []): qry = omniData.find(qryDict)
else: qry = omniData.find()
if (qry.count() > 0):
omniList = []
for rec in qry.sort('time'):
omniList.append(omniRec(dbDict=rec))
logging.info('\nreturning a list with ' + len(omniList) +
' records of omni data')
return omniList
#if we didn't find anything on the mongodb
else:
logging.info('\ncould not find requested data in the mongodb')
logging.info(
'we will look on the ftp server, but your conditions will be (mostly) ignored'
)
#read from ftp server
omniList = readOmniFtp(sTime, eTime)
if (omniList != None):
logging.info('\nreturning a list with ' + len(omniList) +
' recs of omni data')
return omniList
else:
logging.info('\n no data found on FTP server, returning None...')
return None
def readAe(sTime=None, eTime=None, res=60, ae=None, al=None, au=None, ao=None):
"""This function reads ae data from the mongodb. **The data are 1-minute values**
Parameters
----------
sTime : Optional[datetime]
the earliest time you want data for, default=None
eTime : Optional[datetime]
the latest time you want data for. if this is None, end Time will be
1 day after sTime. default = None
res : Optional[int]
the time resolution desired in minutes. Valid inputs are 1 and 60.
default = 60
ae : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with ae values in the range [a,b] will be
returned. default = None
al : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with al values in the range [a,b] will be
returned. default = None
au : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with au values in the range [a,b] will be
returned. default = None
ao : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with ao values in the range [a,b] will be
returned. default = None
Returns
-------
aeList : list
if data is found, a list of class gme.ind.ae.aeRec objects matching
the input parameters is returned. If no data is found, None is
returned.
Example
-------
import datetime as dt
aeList = gme.ind.readAe(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),res=60,ao=[-50,50])
written by AJ, 20130131
"""
import datetime as dt
import davitpy.pydarn.sdio.dbUtils as db
# check all the inputs for validity
assert sTime == None or isinstance(sTime, dt.datetime), logging.error("sTime must be a datetime object")
assert eTime == None or isinstance(eTime, dt.datetime), logging.error(
"eTime must be either None or a datetime object"
)
assert res == 60 or res == 1, logging.error("res must be 1 or 60")
var = locals()
for name in ["ae", "al", "au", "ao"]:
assert var[name] == None or (
isinstance(var[name], list)
and isinstance(var[name][0], (int, float))
and isinstance(var[name][1], (int, float))
), logging.error(name + " must None or a list of 2 numbers")
if eTime == None and sTime != None:
eTime = sTime + dt.timedelta(days=1)
qryList = []
# if arguments are provided, query for those
if sTime != None:
qryList.append({"time": {"$gte": sTime}})
if eTime != None:
qryList.append({"time": {"$lte": eTime}})
qryList.append({"res": res})
var = locals()
for name in ["ae", "al", "au", "ao"]:
if var[name] != None:
qryList.append({name: {"$gte": min(var[name])}})
qryList.append({name: {"$lte": max(var[name])}})
# construct the final query definition
qryDict = {"$and": qryList}
# connect to the database
aeData = db.getDataConn(dbName="gme", collName="ae")
# do the query
if qryList != []:
qry = aeData.find(qryDict)
else:
qry = aeData.find()
if qry.count() > 0:
aeList = []
for rec in qry.sort("time"):
aeList.append(aeRec(dbDict=rec))
logging.info("\nreturning a list with " + str(len(aeList)) + " records of ae data")
return aeList
# if we didn't find anything on the mongodb
else:
logging.info("\ncould not find requested data in the mongodb")
return None
def readOmni(sTime,
eTime=None,
res=5,
bx=None,
bye=None,
bze=None,
bym=None,
bzm=None,
pDyn=None,
ae=None,
symh=None):
"""This function reads omni data. First, it will try to get it from the mongodb, and if it can't find it, it will look on the NASA SPDF FTP server using :func:`readOmniFtp`
**Args**:
* **sTime** (`datetime <http://tinyurl.com/bl352yx>`_ or None): the earliest time you want data for
* [**eTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the latest time you want data for. if this is None, end Time will be 1 day after sTime. default = None
* [**res**] (int): the time reolution of data desired. This can be either 1 or 5. default = 5
* [**bx**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bx values in the range [a,b] will be returned. default = None
* [**bx**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bx values in the range [a,b] will be returned. default = None
* [**bye**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bye values in the range [a,b] will be returned. default = None
* [**bze**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bze values in the range [a,b] will be returned. default = None
* [**bym**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bym values in the range [a,b] will be returned. default = None
* [**bzm**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bzm values in the range [a,b] will be returned. default = None
* [**pDyn**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with pDyn values in the range [a,b] will be returned. default = None
* [**ae**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with ae values in the range [a,b] will be returned. default = None
* [**symh**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with symh values in the range [a,b] will be returned. default = None
**Returns**:
* **omniList** (list or None): if data is found, a list of :class:`omniRec` objects matching the input parameters is returned. If no data is found, None is returned.
**Example**:
::
import datetime as dt
omniList = gme.ind.readOmni(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),bx=[0,5.5],bye=[-1,3.5],bze=[-10,0],ae=[0,56.3])
written by AJ, 20130128
"""
import datetime as dt
import davitpy.pydarn.sdio.dbUtils as db
#check all the inputs for validity
assert(isinstance(sTime,dt.datetime)), \
'error, sTime must be a datetime object'
assert(eTime == None or isinstance(eTime,dt.datetime)), \
'error, eTime must be either None or a datetime object'
assert (res == 5 or res == 1), 'error, res must be either 1 or 5'
var = locals()
for name in ['bx', 'bye', 'bze', 'bym', 'bzm', 'pDyn', 'ae', 'symh']:
assert(var[name] == None or (isinstance(var[name],list) and \
isinstance(var[name][0],(int,float)) and isinstance(var[name][1],(int,float)))), \
'error,'+name+' must None or a list of 2 numbers'
if (eTime == None): eTime = sTime + dt.timedelta(days=1)
qryList = []
#if arguments are provided, query for those
qryList.append({'time': {'$gte': sTime}})
qryList.append({'res': res})
if (eTime != None): qryList.append({'time': {'$lte': eTime}})
var = locals()
for name in ['bx', 'bye', 'bze', 'bym', 'bzm', 'pDyn', 'ae', 'symh']:
if (var[name] != None):
qryList.append({name: {'$gte': min(var[name])}})
qryList.append({name: {'$lte': max(var[name])}})
#construct the final query definition
qryDict = {'$and': qryList}
#connect to the database
omniData = db.getDataConn(dbName='gme', collName='omni')
#do the query
if (qryList != []): qry = omniData.find(qryDict)
else: qry = omniData.find()
if (qry.count() > 0):
omniList = []
for rec in qry.sort('time'):
omniList.append(omniRec(dbDict=rec))
print '\nreturning a list with', len(omniList), 'records of omni data'
return omniList
#if we didn't find anything on the mongodb
else:
print '\ncould not find requested data in the mongodb'
print 'we will look on the ftp server, but your conditions will be (mostly) ignored'
#read from ftp server
omniList = readOmniFtp(sTime, eTime)
if (omniList != None):
print '\nreturning a list with', len(omniList), 'recs of omni data'
return omniList
else:
print '\n no data found on FTP server, returning None...'
return None
def mapAeMongo(sYear, eYear=None, res=60):
"""This function reads ae data from wdc and puts it in mongodb
Parameters
----------
sYear : int
the year to begin mapping data
eYear : Optional[int]
the end year for mapping data. if this is None, eYear will be sYear
res : Optional[int]
the time resolution desired. either 1 or 60 minutes. default=60.
Returns
-------
Nothing
Notes
-----
In general, nobody except the database admins will need to use this function
Example
-------
gme.ind.mapAeMongo(1997)
written by AJ, 20130123
"""
import davitpy.pydarn.sdio.dbUtils as db
from davitpy import rcParams
import datetime as dt
# check inputs
assert isinstance(sYear, int), logging.error("sYear must be int")
if eYear == None:
eYear = sYear
assert isinstance(eYear, int), logging.error("sYear must be None or int")
assert eYear >= sYear, logging.error("end year less than than start year")
# get data connection
mongoData = db.getDataConn(
username=rcParams["DBWRITEUSER"],
password=rcParams["DBWRITEPASS"],
dbAddress=rcParams["SDDB"],
dbName="gme",
collName="ae",
)
# set up all of the indices
mongoData.ensure_index("time")
mongoData.ensure_index("ae")
mongoData.ensure_index("al")
mongoData.ensure_index("au")
mongoData.ensure_index("ao")
mongoData.ensure_index("res")
for yr in range(sYear, eYear + 1):
# 1 day at a time, to not fill up RAM
templist = readAeWeb(dt.datetime(yr, 1, 1), dt.datetime(yr, 1, 1) + dt.timedelta(days=366), res=res)
if templist == None:
continue
for rec in templist:
# check if a duplicate record exists
qry = mongoData.find({"$and": [{"time": rec.time}, {"res": rec.res}]})
logging.debug(rec.time)
tempRec = rec.toDbDict()
cnt = qry.count()
# if this is a new record, insert it
if cnt == 0:
mongoData.insert(tempRec)
# if this is an existing record, update it
elif cnt == 1:
logging.debug("foundone!!")
dbDict = qry.next()
temp = dbDict["_id"]
dbDict = tempRec
dbDict["_id"] = temp
mongoData.save(dbDict)
else:
logging.warning("strange, there is more than 1 AE record for " + rec.time)
del templist
def readAe(sTime=None,eTime=None,res=60,ae=None,al=None,au=None,ao=None):
"""This function reads ae data from the mongodb. **The data are 1-minute values**
**Args**:
* [**sTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the earliest time you want data for, default=None
* [**eTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the latest time you want data for. if this is None, end Time will be 1 day after sTime. default = None
* [**res**] (int): the time resolution desired in minutes. Valid inputs are 1 and 60. default = 60
* [**ae**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with ae values in the range [a,b] will be returned. default = None
* [**al**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with al values in the range [a,b] will be returned. default = None
* [**au**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with au values in the range [a,b] will be returned. default = None
* [**ao**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with ao values in the range [a,b] will be returned. default = None
**Returns**:
* **aeList** (list or None): if data is found, a list of :class:`gme.ind.ae.aeRec` objects matching the input parameters is returned. If no data is found, None is returned.
**Example**:
::
import datetime as dt
aeList = gme.ind.readAe(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),res=60,ao=[-50,50])
written by AJ, 20130131
"""
import datetime as dt
import davitpy.pydarn.sdio.dbUtils as db
#check all the inputs for validity
assert(sTime == None or isinstance(sTime,dt.datetime)), \
'error, sTime must be a datetime object'
assert(eTime == None or isinstance(eTime,dt.datetime)), \
'error, eTime must be either None or a datetime object'
assert(res == 60 or res ==1), 'error, res must be 1 or 60'
var = locals()
for name in ['ae','al','au','ao']:
assert(var[name] == None or (isinstance(var[name],list) and \
isinstance(var[name][0],(int,float)) and isinstance(var[name][1],(int,float)))), \
'error,'+name+' must None or a list of 2 numbers'
if(eTime == None and sTime != None): eTime = sTime+dt.timedelta(days=1)
qryList = []
#if arguments are provided, query for those
if(sTime != None): qryList.append({'time':{'$gte':sTime}})
if(eTime != None): qryList.append({'time':{'$lte':eTime}})
qryList.append({'res':res})
var = locals()
for name in ['ae','al','au','ao']:
if(var[name] != None):
qryList.append({name:{'$gte':min(var[name])}})
qryList.append({name:{'$lte':max(var[name])}})
#construct the final query definition
qryDict = {'$and': qryList}
#connect to the database
aeData = db.getDataConn(dbName='gme',collName='ae')
#do the query
if(qryList != []): qry = aeData.find(qryDict)
else: qry = aeData.find()
if(qry.count() > 0):
aeList = []
for rec in qry.sort('time'):
aeList.append(aeRec(dbDict=rec))
print '\nreturning a list with',len(aeList),'records of ae data'
return aeList
#if we didn't find anything on the mongodb
else:
print '\ncould not find requested data in the mongodb'
return None
def readOmni(sTime,eTime=None,res=5,bx=None,bye=None,bze=None,bym=None,bzm=None,pDyn=None,ae=None,symh=None):
"""This function reads omni data. First, it will try to get it from the
mongodb, and if it can't find it, it will look on the NASA SPDF FTP
server using readOmniFtp
Parameters
----------
sTime : datetime
the earliest time you want data for
eTime : Optional[datetime]
the latest time you want data for. if this is None, end Time will
be 1 day after sTime. default = None
res : Optional[int]
the time reolution of data desired. This can be either 1
or 5. default = 5
bx : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with bx values in the range [a,b] will be
returned. default = None
bx : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with bx values in the range [a,b] will be
returned. default = None
bye : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with bye values in the range [a,b] will be
returned. default = None
bze : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with bze values in the range [a,b] will be
returned. default = None
bym : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with bym values in the range [a,b] will be
returned. default = None
bzm : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with bzm values in the range [a,b] will be
returned. default = None
pDyn : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with pDyn values in the range [a,b] will be
returned. default = None
ae : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with ae values in the range [a,b] will be
returned. default = None
symh : Optional[list]
if this is not None, it must be a 2-element list of numbers, [a,b].
In this case, only data with symh values in the range [a,b] will be
returned. default = None
Returns
-------
omniList : list
if data is found, a list of class omniRec objects matching the
input parameters is returned. If no data is found, None is
returned.
Example
-------
import datetime as dt
omniList = gme.ind.readOmni(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),bx=[0,5.5],bye=[-1,3.5],bze=[-10,0],ae=[0,56.3])
written by AJ, 20130128
"""
import datetime as dt
import davitpy.pydarn.sdio.dbUtils as db
#check all the inputs for validity
assert(isinstance(sTime,dt.datetime)), \
logging.error('sTime must be a datetime object')
assert(eTime == None or isinstance(eTime,dt.datetime)), \
logging.error('eTime must be either None or a datetime object')
assert(res==5 or res == 1), logging.error('res must be either 1 or 5')
var = locals()
for name in ['bx','bye','bze','bym','bzm','pDyn','ae','symh']:
assert(var[name] == None or (isinstance(var[name],list) and \
isinstance(var[name][0],(int,float)) and isinstance(var[name][1],(int,float)))), \
logging.error(name + ' must None or a list of 2 numbers')
if(eTime == None): eTime = sTime+dt.timedelta(days=1)
qryList = []
#if arguments are provided, query for those
qryList.append({'time':{'$gte':sTime}})
qryList.append({'res':res})
if(eTime != None): qryList.append({'time':{'$lte':eTime}})
var = locals()
for name in ['bx','bye','bze','bym','bzm','pDyn','ae','symh']:
if(var[name] != None):
qryList.append({name:{'$gte':min(var[name])}})
qryList.append({name:{'$lte':max(var[name])}})
#construct the final query definition
qryDict = {'$and': qryList}
#connect to the database
omniData = db.getDataConn(dbName='gme',collName='omni')
#do the query
if(qryList != []): qry = omniData.find(qryDict)
else: qry = omniData.find()
if(qry.count() > 0):
omniList = []
for rec in qry.sort('time'):
omniList.append(omniRec(dbDict=rec))
logging.info('\nreturning a list with ' + len(omniList) + ' records of omni data')
return omniList
#if we didn't find anything on the mongodb
else:
logging.info('\ncould not find requested data in the mongodb')
logging.info('we will look on the ftp server, but your conditions will be (mostly) ignored')
#read from ftp server
omniList = readOmniFtp(sTime, eTime)
if(omniList != None):
logging.info('\nreturning a list with ' + len(omniList) + ' recs of omni data')
return omniList
else:
logging.info('\n no data found on FTP server, returning None...')
return None
def mapAeMongo(sYear, eYear=None, res=60):
"""This function reads ae data from wdc and puts it in mongodb
Parameters
----------
sYear : int
the year to begin mapping data
eYear : Optional[int]
the end year for mapping data. if this is None, eYear will be sYear
res : Optional[int]
the time resolution desired. either 1 or 60 minutes. default=60.
Returns
-------
Nothing
Notes
-----
In general, nobody except the database admins will need to use this function
Example
-------
gme.ind.mapAeMongo(1997)
written by AJ, 20130123
"""
import davitpy.pydarn.sdio.dbUtils as db
from davitpy import rcParams
import datetime as dt
#check inputs
assert (isinstance(sYear, int)), logging.error('sYear must be int')
if (eYear == None): eYear = sYear
assert (isinstance(eYear, int)), logging.error('sYear must be None or int')
assert (eYear >=
sYear), logging.error('end year less than than start year')
#get data connection
mongoData = db.getDataConn(username=rcParams['DBWRITEUSER'],password=rcParams['DBWRITEPASS'],\
dbAddress=rcParams['SDDB'],dbName='gme',collName='ae')
#set up all of the indices
mongoData.ensure_index('time')
mongoData.ensure_index('ae')
mongoData.ensure_index('al')
mongoData.ensure_index('au')
mongoData.ensure_index('ao')
mongoData.ensure_index('res')
for yr in range(sYear, eYear + 1):
#1 day at a time, to not fill up RAM
templist = readAeWeb(dt.datetime(yr, 1, 1),
dt.datetime(yr, 1, 1) + dt.timedelta(days=366),
res=res)
if (templist == None): continue
for rec in templist:
#check if a duplicate record exists
qry = mongoData.find(
{'$and': [{
'time': rec.time
}, {
'res': rec.res
}]})
logging.debug(rec.time)
tempRec = rec.toDbDict()
cnt = qry.count()
#if this is a new record, insert it
if (cnt == 0):
mongoData.insert(tempRec)
#if this is an existing record, update it
elif (cnt == 1):
logging.debug('foundone!!')
dbDict = qry.next()
temp = dbDict['_id']
dbDict = tempRec
dbDict['_id'] = temp
mongoData.save(dbDict)
else:
logging.warning(
'strange, there is more than 1 AE record for ' + rec.time)
del templist
def readPoes(sTime,eTime=None,satnum=None,folat=None,folon=None,ted=None,echar=None,pchar=None):
"""This function reads poes data. First, it will try to get it from the mongodb, and if it can't find it, it will look on the NOAA NGDC FTP server using :func:`readPoesFtp`. The data are 16-second averages
**Args**:
* **sTime** (`datetime <http://tinyurl.com/bl352yx>`_ or None): the earliest time you want data for
* [**eTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the latest time you want data for. if this is None, end Time will be 1 day after sTime. default = None
* [**satnum**] (int): the satellite you want data for. eg 17 for noaa17. if this is None, data for all satellites will be returned. default = None
* [**satnum**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bx values in the range [a,b] will be returned. default = None
* [**folat**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bx values in the range [a,b] will be returned. default = None
* [**folon**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bye values in the range [a,b] will be returned. default = None
* [**ted**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bze values in the range [a,b] will be returned. default = None
* [**echar**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bym values in the range [a,b] will be returned. default = None
* [**pchar**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bzm values in the range [a,b] will be returned. default = None
**Returns**:
* **poesList** (list or None): if data is found, a list of :class:`poesRec` objects matching the input parameters is returned. If no data is found, None is returned.
**Example**:
::
import datetime as dt
poesList = gme.sat.readPoes(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),folat=[60,80])
written by AJ, 20130131
"""
import datetime as dt
import davitpy.pydarn.sdio.dbUtils as db
#check all the inputs for validity
assert(isinstance(sTime,dt.datetime)), \
'error, sTime must be a datetime object'
assert(eTime == None or isinstance(eTime,dt.datetime)), \
'error, eTime must be either None or a datetime object'
assert(satnum == None or isinstance(satnum,int)), 'error, satnum must be an int'
var = locals()
for name in ['folat','folon','ted','echar','pchar']:
assert(var[name] == None or (isinstance(var[name],list) and \
isinstance(var[name][0],(int,float)) and isinstance(var[name][1],(int,float)))), \
'error,'+name+' must None or a list of 2 numbers'
if(eTime == None): eTime = sTime+dt.timedelta(days=1)
qryList = []
#if arguments are provided, query for those
qryList.append({'time':{'$gte':sTime}})
if(eTime != None): qryList.append({'time':{'$lte':eTime}})
if(satnum != None): qryList.append({'satnum':satnum})
var = locals()
for name in ['folat','folon','ted','echar','pchar']:
if(var[name] != None):
qryList.append({name:{'$gte':min(var[name])}})
qryList.append({name:{'$lte':max(var[name])}})
#construct the final query definition
qryDict = {'$and': qryList}
#connect to the database
poesData = db.getDataConn(dbName='gme',collName='poes')
#do the query
if(qryList != []): qry = poesData.find(qryDict)
else: qry = poesData.find()
if(qry.count() > 0):
poesList = []
for rec in qry.sort('time'):
poesList.append(poesRec(dbDict=rec))
print '\nreturning a list with',len(poesList),'records of poes data'
return poesList
#if we didn't find anything on the mongodb
else:
print '\ncould not find requested data in the mongodb'
return None
def mapPoesMongo(syear, eYear=None):
"""This function reads poes data from the NOAA NGDC FTP server via anonymous
FTP connection and maps it to the mongodb.
Parameters
----------
syear : (int)
the year to begin mapping data
eYear : (int or None)
the end year for mapping data. if this is None, eYear will be syear
Returns
-------
Nothing.
Notes
-----
In general, nobody except the database admins will need to use this function
Example
-------
gme.sat.mapPoesMongo(2004)
written by AJ, 20130131
"""
import davitpy.pydarn.sdio.dbUtils as db
from davitpy import rcParams
import datetime as dt
# check inputs
assert isinstance(syear, int), logging.error('syear must be int')
if eYear == None:
eYear = syear
assert isinstance(eYear, int), logging.error('syear must be None or int')
assert eYear >= syear, logging.error('end year greater than start year')
# get data connection
mongo_data = db.getDataConn(username=rcParams['DBWRITEUSER'],
password=rcParams['DBWRITEPASS'],
dbAddress=rcParams['SDDB'], dbName='gme',
collName='poes')
# set up all of the indices
mongo_data.ensure_index('time')
mongo_data.ensure_index('satnum')
mongo_data.ensure_index('folat')
mongo_data.ensure_index('folon')
mongo_data.ensure_index('ted')
mongo_data.ensure_index('echar')
mongo_data.ensure_index('pchar')
# read the poes data from the FTP server
my_time = dt.datetime(syear, 1, 1)
while my_time < dt.datetime(eYear+1, 1, 1):
# 10 day at a time, to not fill up RAM
templist = readPoesFtp(my_time, my_time+dt.timedelta(days=10))
if(templist == None):
continue
for rec in templist:
# check if a duplicate record exists
qry = mongo_data.find({'$and':[{'time':rec.time},
{'satnum':rec.satnum}]})
logging.info("{:}, {:}".format(rec.time, rec.satnum))
tempRec = rec.toDbDict()
cnt = qry.count()
# if this is a new record, insert it; if it exists, update it
if cnt == 0:
mongo_data.insert(tempRec)
elif cnt == 1:
logging.debug('found one!!')
db_dict = qry.next()
temp = db_dict['_id']
db_dict = tempRec
db_dict['_id'] = temp
mongo_data.save(db_dict)
else:
logging.info('strange, there is more than 1 record for '
+ rec.time)
del templist
my_time += dt.timedelta(days=10)
def mapPoesMongo(sYear,eYear=None):
"""This function reads poes data from the NOAA NGDC FTP server via anonymous FTP connection and maps it to the mongodb.
.. warning::
In general, nobody except the database admins will need to use this function
**Args**:
* **sYear** (int): the year to begin mapping data
* [**eYear**] (int or None): the end year for mapping data. if this is None, eYear will be sYear
**Returns**:
* Nothing.
**Example**:
::
gme.sat.mapPoesMongo(2004)
written by AJ, 20130131
"""
import davitpy.pydarn.sdio.dbUtils as db
from davitpy import rcParams
import datetime as dt
#check inputs
assert(isinstance(sYear,int)),'error, sYear must be int'
if(eYear == None): eYear=sYear
assert(isinstance(eYear,int)),'error, sYear must be None or int'
assert(eYear >= sYear), 'error, end year greater than start year'
#get data connection
mongoData = db.getDataConn(username=rcParams['DBWRITEUSER'],password=rcParams['DBWRITEPASS'],\
dbAddress=rcParams['SDDB'],dbName='gme',collName='poes')
#set up all of the indices
mongoData.ensure_index('time')
mongoData.ensure_index('satnum')
mongoData.ensure_index('folat')
mongoData.ensure_index('folon')
mongoData.ensure_index('ted')
mongoData.ensure_index('echar')
mongoData.ensure_index('pchar')
#read the poes data from the FTP server
myTime = dt.datetime(sYear,1,1)
while(myTime < dt.datetime(eYear+1,1,1)):
#10 day at a time, to not fill up RAM
templist = readPoesFtp(myTime,myTime+dt.timedelta(days=10))
if(templist == None): continue
for rec in templist:
#check if a duplicate record exists
qry = mongoData.find({'$and':[{'time':rec.time},{'satnum':rec.satnum}]})
print rec.time, rec.satnum
tempRec = rec.toDbDict()
cnt = qry.count()
#if this is a new record, insert it
if(cnt == 0): mongoData.insert(tempRec)
#if this is an existing record, update it
elif(cnt == 1):
print 'foundone!!'
dbDict = qry.next()
temp = dbDict['_id']
dbDict = tempRec
dbDict['_id'] = temp
mongoData.save(dbDict)
else:
print 'strange, there is more than 1 record for',rec.time
del templist
myTime += dt.timedelta(days=10)
