def mapKpMongo(sYear, eYear=None):
"""This function reads kp data from the GFZ Potsdam FTP server via anonymous FTP connection and maps it to the mongodb.
.. warning::
In general, nobody except the database admins will need to use this function
**Args**:
* **sYear** (int): the year to begin mapping data
* [**eYear**] (int or None): the end year for mapping data. if this is None, eYear will be sYear. default=None
**Returns**:
* Nothing.
**Example**:
::
gme.ind.mapKpMongo(1985,eTime=1986)
written by AJ, 20130123
"""
import pydarn.sdio.dbUtils as db
import os, datetime as dt
if (eYear == None): eYear = sYear
assert (eYear >= sYear), 'error, end year greater than start year'
mongoData = db.getDataConn(username=os.environ['DBWRITEUSER'],password=os.environ['DBWRITEPASS'],\
dbAddress=os.environ['SDDB'],dbName='gme',collName='kp')
#set up all of the indices
mongoData.ensure_index('time')
mongoData.ensure_index('kp')
mongoData.ensure_index('ap')
mongoData.ensure_index('kpSum')
mongoData.ensure_index('apMean')
mongoData.ensure_index('sunspot')
#read the kp data from the FTP server
datalist = []
for yr in range(sYear, eYear + 1):
templist = readKpFtp(dt.datetime(yr, 1, 1), dt.datetime(yr + 1, 1, 1))
if (templist == None): continue
for rec in templist:
#check if a duplicate record exists
qry = mongoData.find({'time': rec.time})
tempRec = rec.toDbDict()
cnt = qry.count()
#if this is a new record, insert it
if (cnt == 0):
mongoData.insert(tempRec)
#if this is an existing record, update it
elif (cnt == 1):
print 'foundone!!'
dbDict = qry.next()
temp = dbDict['_id']
dbDict = tempRec
dbDict['_id'] = temp
mongoData.save(dbDict)
else:
print 'strange, there is more than 1 record for', rec.time
def mapDstMongo(sYear,eYear=None):
"""This function reads dst data from wdc and puts it in mongodb
.. warning::
In general, nobody except the database admins will need to use this function
**Args**:
* **sYear** (int): the year to begin mapping data
* [**eYear**] (int or None): the end year for mapping data. if this is None, eYear will be sYear
**Returns**:
* Nothing.
**Example**:
::
gme.ind.mapDstMongo(1997)
written by AJ, 20130123
"""
import pydarn.sdio.dbUtils as db
import os, datetime as dt
#check inputs
assert(isinstance(sYear,int)),'error, sYear must be int'
if(eYear == None): eYear=sYear
assert(isinstance(eYear,int)),'error, sYear must be None or int'
assert(eYear >= sYear), 'error, end year greater than start year'
#get data connection
mongoData = db.getDataConn(username=os.environ['DBWRITEUSER'],password=os.environ['DBWRITEPASS'],\
dbAddress=os.environ['SDDB'],dbName='gme',collName='dst')
#set up all of the indices
mongoData.ensure_index('time')
mongoData.ensure_index('dst')
for yr in range(sYear,eYear+1):
#1 year at a time, to not fill up RAM
templist = readDstWeb(dt.datetime(yr,1,1),dt.datetime(yr,12,31))
for rec in templist:
#check if a duplicate record exists
qry = mongoData.find({'time':rec.time})
print rec.time
tempRec = rec.toDbDict()
cnt = qry.count()
#if this is a new record, insert it
if(cnt == 0): mongoData.insert(tempRec)
#if this is an existing record, update it
elif(cnt == 1):
print 'foundone!!'
dbDict = qry.next()
temp = dbDict['_id']
dbDict = tempRec
dbDict['_id'] = temp
mongoData.save(dbDict)
else:
print 'strange, there is more than 1 DST record for',rec.time
del templist
def mapKpMongo(sYear,eYear=None):
"""This function reads kp data from the GFZ Potsdam FTP server via anonymous FTP connection and maps it to the mongodb.
.. warning::
In general, nobody except the database admins will need to use this function
**Args**:
* **sYear** (int): the year to begin mapping data
* [**eYear**] (int or None): the end year for mapping data. if this is None, eYear will be sYear. default=None
**Returns**:
* Nothing.
**Example**:
::
gme.ind.mapKpMongo(1985,eTime=1986)
written by AJ, 20130123
"""
import pydarn.sdio.dbUtils as db
import os, datetime as dt
if(eYear == None): eYear=sYear
assert(eYear >= sYear), 'error, end year greater than start year'
mongoData = db.getDataConn(username=os.environ['DBWRITEUSER'],password=os.environ['DBWRITEPASS'],\
dbAddress=os.environ['SDDB'],dbName='gme',collName='kp')
#set up all of the indices
mongoData.ensure_index('time')
mongoData.ensure_index('kp')
mongoData.ensure_index('ap')
mongoData.ensure_index('kpSum')
mongoData.ensure_index('apMean')
mongoData.ensure_index('sunspot')
#read the kp data from the FTP server
datalist = []
for yr in range(sYear,eYear+1):
templist = readKpFtp(dt.datetime(yr,1,1), dt.datetime(yr+1,1,1))
if(templist == None): continue
for rec in templist:
#check if a duplicate record exists
qry = mongoData.find({'time':rec.time})
tempRec = rec.toDbDict()
cnt = qry.count()
#if this is a new record, insert it
if(cnt == 0): mongoData.insert(tempRec)
#if this is an existing record, update it
elif(cnt == 1):
print 'foundone!!'
dbDict = qry.next()
temp = dbDict['_id']
dbDict = tempRec
dbDict['_id'] = temp
mongoData.save(dbDict)
else:
print 'strange, there is more than 1 record for',rec.time
def readDst(sTime=None,eTime=None,dst=None):
"""This function reads dst data from the mongodb.
**Args**:
* [**sTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the earliest time you want data for, default=None
* [**eTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the latest time you want data for. if this is None, end Time will be 1 day after sTime. default = None
* [**dst**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with dst values in the range [a,b] will be returned. default = None
**Returns**:
* **dstList** (list or None): if data is found, a list of :class:`gme.ind.dst.dstRec` objects matching the input parameters is returned. If no data is found, None is returned.
**Example**:
::
import datetime as dt
dstList = gme.ind.readDst(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),dst=[-50,50])
written by AJ, 20130131
"""
import datetime as dt
import pydarn.sdio.dbUtils as db
#check all the inputs for validity
assert(sTime == None or isinstance(sTime,dt.datetime)), \
'error, sTime must be a datetime object'
assert(eTime == None or isinstance(eTime,dt.datetime)), \
'error, eTime must be either None or a datetime object'
assert(dst == None or (isinstance(dst,list) and \
isinstance(dst[0],(int,float)) and isinstance(dst[1],(int,float)))), \
'error,dst must None or a list of 2 numbers'
if(eTime == None and sTime != None): eTime = sTime+dt.timedelta(days=1)
qryList = []
#if arguments are provided, query for those
if(sTime != None): qryList.append({'time':{'$gte':sTime}})
if(eTime != None): qryList.append({'time':{'$lte':eTime}})
if(dst != None):
qryList.append({'dst':{'$gte':min(dst)}})
qryList.append({'dst':{'$lte':max(dst)}})
#construct the final query definition
qryDict = {'$and': qryList}
#connect to the database
dstData = db.getDataConn(dbName='gme',collName='dst')
#do the query
if(qryList != []): qry = dstData.find(qryDict)
else: qry = dstData.find()
if(qry.count() > 0):
dstList = []
for rec in qry.sort('time'):
dstList.append(dstRec(dbDict=rec))
print '\nreturning a list with',len(dstList),'records of dst data'
return dstList
#if we didn't find anything on the mongodb
else:
print '\ncould not find requested data in the mongodb'
return None
def mapPoesMongo(sYear,eYear=None):
"""This function reads poes data from the NOAA NGDC FTP server via anonymous FTP connection and maps it to the mongodb.
.. warning::
In general, nobody except the database admins will need to use this function
**Args**:
* **sYear** (int): the year to begin mapping data
* [**eYear**] (int or None): the end year for mapping data. if this is None, eYear will be sYear
**Returns**:
* Nothing.
**Example**:
::
gme.sat.mapPoesMongo(2004)
written by AJ, 20130131
"""
import pydarn.sdio.dbUtils as db
import os, datetime as dt
#check inputs
assert(isinstance(sYear,int)),'error, sYear must be int'
if(eYear == None): eYear=sYear
assert(isinstance(eYear,int)),'error, sYear must be None or int'
assert(eYear >= sYear), 'error, end year greater than start year'
#get data connection
mongoData = db.getDataConn(username=os.environ['DBWRITEUSER'],password=os.environ['DBWRITEPASS'],\
dbAddress=os.environ['SDDB'],dbName='gme',collName='poes')
#set up all of the indices
mongoData.ensure_index('time')
mongoData.ensure_index('satnum')
mongoData.ensure_index('folat')
mongoData.ensure_index('folon')
mongoData.ensure_index('ted')
mongoData.ensure_index('echar')
mongoData.ensure_index('pchar')
#read the poes data from the FTP server
myTime = dt.datetime(sYear,1,1)
while(myTime < dt.datetime(eYear+1,1,1)):
#10 day at a time, to not fill up RAM
templist = readPoesFtp(myTime,myTime+dt.timedelta(days=10))
if(templist == None): continue
for rec in templist:
#check if a duplicate record exists
qry = mongoData.find({'$and':[{'time':rec.time},{'satnum':rec.satnum}]})
print rec.time, rec.satnum
tempRec = rec.toDbDict()
cnt = qry.count()
#if this is a new record, insert it
if(cnt == 0): mongoData.insert(tempRec)
#if this is an existing record, update it
elif(cnt == 1):
print 'foundone!!'
dbDict = qry.next()
temp = dbDict['_id']
dbDict = tempRec
dbDict['_id'] = temp
mongoData.save(dbDict)
else:
print 'strange, there is more than 1 record for',rec.time
del templist
myTime += dt.timedelta(days=10)
def readPoes(sTime,eTime=None,satnum=None,folat=None,folon=None,ted=None,echar=None,pchar=None):
"""This function reads poes data. First, it will try to get it from the mongodb, and if it can't find it, it will look on the NOAA NGDC FTP server using :func:`readPoesFtp`. The data are 16-second averages
**Args**:
* **sTime** (`datetime <http://tinyurl.com/bl352yx>`_ or None): the earliest time you want data for
* [**eTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the latest time you want data for. if this is None, end Time will be 1 day after sTime. default = None
* [**satnum**] (int): the satellite you want data for. eg 17 for noaa17. if this is None, data for all satellites will be returned. default = None
* [**satnum**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bx values in the range [a,b] will be returned. default = None
* [**folat**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bx values in the range [a,b] will be returned. default = None
* [**folon**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bye values in the range [a,b] will be returned. default = None
* [**ted**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bze values in the range [a,b] will be returned. default = None
* [**echar**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bym values in the range [a,b] will be returned. default = None
* [**pchar**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bzm values in the range [a,b] will be returned. default = None
**Returns**:
* **poesList** (list or None): if data is found, a list of :class:`poesRec` objects matching the input parameters is returned. If no data is found, None is returned.
**Example**:
::
import datetime as dt
poesList = gme.sat.readPoes(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),folat=[60,80])
written by AJ, 20130131
"""
import datetime as dt
import pydarn.sdio.dbUtils as db
#check all the inputs for validity
assert(isinstance(sTime,dt.datetime)), \
'error, sTime must be a datetime object'
assert(eTime == None or isinstance(eTime,dt.datetime)), \
'error, eTime must be either None or a datetime object'
assert(satnum == None or isinstance(satnum,int)), 'error, satnum must be an int'
var = locals()
for name in ['folat','folon','ted','echar','pchar']:
assert(var[name] == None or (isinstance(var[name],list) and \
isinstance(var[name][0],(int,float)) and isinstance(var[name][1],(int,float)))), \
'error,'+name+' must None or a list of 2 numbers'
if(eTime == None): eTime = sTime+dt.timedelta(days=1)
qryList = []
#if arguments are provided, query for those
qryList.append({'time':{'$gte':sTime}})
if(eTime != None): qryList.append({'time':{'$lte':eTime}})
if(satnum != None): qryList.append({'satnum':satnum})
var = locals()
for name in ['folat','folon','ted','echar','pchar']:
if(var[name] != None):
qryList.append({name:{'$gte':min(var[name])}})
qryList.append({name:{'$lte':max(var[name])}})
#construct the final query definition
qryDict = {'$and': qryList}
#connect to the database
poesData = db.getDataConn(dbName='gme',collName='poes')
#do the query
if(qryList != []): qry = poesData.find(qryDict)
else: qry = poesData.find()
if(qry.count() > 0):
poesList = []
for rec in qry.sort('time'):
poesList.append(poesRec(dbDict=rec))
print '\nreturning a list with',len(poesList),'records of poes data'
return poesList
#if we didn't find anything on the mongodb
else:
print '\ncould not find requested data in the mongodb'
return None
def mapAeMongo(sYear,eYear=None,res=60):
"""This function reads ae data from wdc and puts it in mongodb
.. warning::
In general, nobody except the database admins will need to use this function
**Args**:
* **sYear** (int): the year to begin mapping data
* [**eYear**] (int or None): the end year for mapping data. if this is None, eYear will be sYear
* [**res**] (int): the time resolution desired. either 1 or 60 minutes. default=60.
**Returns**:
* Nothing.
**Example**:
::
gme.ind.mapAeMongo(1997)
written by AJ, 20130123
"""
import pydarn.sdio.dbUtils as db
import os, datetime as dt
#check inputs
assert(isinstance(sYear,int)),'error, sYear must be int'
if(eYear == None): eYear=sYear
assert(isinstance(eYear,int)),'error, sYear must be None or int'
assert(eYear >= sYear), 'error, end year less than than start year'
#get data connection
mongoData = db.getDataConn(username=os.environ['DBWRITEUSER'],password=os.environ['DBWRITEPASS'],\
dbAddress=os.environ['SDDB'],dbName='gme',collName='ae')
#set up all of the indices
mongoData.ensure_index('time')
mongoData.ensure_index('ae')
mongoData.ensure_index('al')
mongoData.ensure_index('au')
mongoData.ensure_index('ao')
mongoData.ensure_index('res')
for yr in range(sYear,eYear+1):
#1 day at a time, to not fill up RAM
templist = readAeWeb(dt.datetime(yr,1,1),dt.datetime(yr,1,1)+dt.timedelta(days=366),res=res)
if(templist == None): continue
for rec in templist:
#check if a duplicate record exists
qry = mongoData.find({'$and':[{'time': rec.time}, {'res': rec.res}]})
print rec.time
tempRec = rec.toDbDict()
cnt = qry.count()
#if this is a new record, insert it
if(cnt == 0): mongoData.insert(tempRec)
#if this is an existing record, update it
elif(cnt == 1):
print 'foundone!!'
dbDict = qry.next()
temp = dbDict['_id']
dbDict = tempRec
dbDict['_id'] = temp
mongoData.save(dbDict)
else:
print 'strange, there is more than 1 AE record for',rec.time
del templist
def readAe(sTime=None,eTime=None,res=60,ae=None,al=None,au=None,ao=None):
"""This function reads ae data from the mongodb. **The data are 1-minute values**
**Args**:
* [**sTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the earliest time you want data for, default=None
* [**eTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the latest time you want data for. if this is None, end Time will be 1 day after sTime. default = None
* [**res**] (int): the time resolution desired in minutes. Valid inputs are 1 and 60. default = 60
* [**ae**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with ae values in the range [a,b] will be returned. default = None
* [**al**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with al values in the range [a,b] will be returned. default = None
* [**au**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with au values in the range [a,b] will be returned. default = None
* [**ao**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with ao values in the range [a,b] will be returned. default = None
**Returns**:
* **aeList** (list or None): if data is found, a list of :class:`gme.ind.ae.aeRec` objects matching the input parameters is returned. If no data is found, None is returned.
**Example**:
::
import datetime as dt
aeList = gme.ind.readAe(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),res=60,ao=[-50,50])
written by AJ, 20130131
"""
import datetime as dt
import pydarn.sdio.dbUtils as db
#check all the inputs for validity
assert(sTime == None or isinstance(sTime,dt.datetime)), \
'error, sTime must be a datetime object'
assert(eTime == None or isinstance(eTime,dt.datetime)), \
'error, eTime must be either None or a datetime object'
assert(res == 60 or res ==1), 'error, res must be 1 or 60'
var = locals()
for name in ['ae','al','au','ao']:
assert(var[name] == None or (isinstance(var[name],list) and \
isinstance(var[name][0],(int,float)) and isinstance(var[name][1],(int,float)))), \
'error,'+name+' must None or a list of 2 numbers'
if(eTime == None and sTime != None): eTime = sTime+dt.timedelta(days=1)
qryList = []
#if arguments are provided, query for those
if(sTime != None): qryList.append({'time':{'$gte':sTime}})
if(eTime != None): qryList.append({'time':{'$lte':eTime}})
qryList.append({'res':res})
var = locals()
for name in ['ae','al','au','ao']:
if(var[name] != None):
qryList.append({name:{'$gte':min(var[name])}})
qryList.append({name:{'$lte':max(var[name])}})
#construct the final query definition
qryDict = {'$and': qryList}
#connect to the database
aeData = db.getDataConn(dbName='gme',collName='ae')
#do the query
if(qryList != []): qry = aeData.find(qryDict)
else: qry = aeData.find()
if(qry.count() > 0):
aeList = []
for rec in qry.sort('time'):
aeList.append(aeRec(dbDict=rec))
print '\nreturning a list with',len(aeList),'records of ae data'
return aeList
#if we didn't find anything on the mongodb
else:
print '\ncould not find requested data in the mongodb'
return None
def readKp(sTime=None,
eTime=None,
kpMin=None,
apMin=None,
kpSum=None,
apMean=None,
sunspot=None):
"""This function reads kp data. First, it will try to get it from the mongodb, and if it can't find it, it will look on the GFZ ftp server using :func:`gme.ind.kp.readKpFtp`
**Args**:
* [**sTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the earliest time you want data for. if this is None, start time will be the earliest record found. default=None
* [**eTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the latest time you want data for. if this is None, end Time will be latest record found. default=None
* [**kpMin**] (int or None): specify this to only return data from dates with a 3-hour kp value of minimum kpMin. if this is none, it will be ignored. default=None
* [**apMin**] (int or None): specify this to only return data from dates with a 3-hour ap value of minimum apMin. if this is none, it will be ignored. default=None
* [**kpSum**] (list or None): this must be a 2 element list of integers. if this is specified, only dates with kpSum values in the range [a,b] will be returned. if this is None, it will be ignored. default=None
* [**apMean**] (list or None): this must be a 2 element list of integers. if this is specified, only dates with apMean values in the range [a,b] will be returned. if this is None, it will be ignored. default=None
* [**sunspot**] (list or None): this must be a 2 element list of integers. if this is specified, only dates with sunspot values in the range [a,b] will be returned. if this is None, it will be ignored. default=None
**Returns**:
* **kpList** (list or None): if data is found, a list of :class:`gme.ind.kp.kpDay` objects matching the input parameters is returned. If not data is found, None is returned.
**Example**:
::
import datetime as dt
kpList = gme.ind.readKp(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),kpMin=2,apMin=1,kpSum=[0,10],apMean=[0,50],sunspot=[6,100])
written by AJ, 20130123
"""
import datetime as dt
import pydarn.sdio.dbUtils as db
#check all the inputs for validity
assert(sTime == None or isinstance(sTime,dt.datetime)), \
'error, sTime must be either None or a datetime object'
assert(eTime == None or isinstance(eTime,dt.datetime)), \
'error, eTime must be either None or a datetime object'
assert(kpMin == None or isinstance(kpMin,int)), \
'error, kpMin must be either None or an int'
assert(apMin == None or isinstance(apMin,int)), \
'error, apMin must be either None or an int'
assert(kpSum == None or (isinstance(kpSum,list) and len(kpSum) == 2 and \
isinstance(kpSum[0], int) and isinstance(kpSum[1], int))), \
'error, kpSum must be either None or a 2 element list'
assert(apMean == None or (isinstance(apMean,list) and len(apMean) == 2and \
isinstance(apMean[0], int) and isinstance(apMean[1], int))), \
'error, apMean must be either None or a 2 element list'
assert(sunspot == None or (isinstance(sunspot,list) and len(sunspot) == 2and \
isinstance(sunspot[0], int) and isinstance(sunspot[1], int))), \
'error, sunspot must be either None or a 2 element list'
qryList = []
#if arguments are provided, query for those
if (sTime != None): qryList.append({'time': {'$gte': sTime}})
if (eTime != None): qryList.append({'time': {'$lte': eTime}})
if (kpMin != None): qryList.append({'kp': {'$gte': kpMin}})
if (apMin != None): qryList.append({'ap': {'$gte': kpMin}})
if (kpSum != None): qryList.append({'kpSum': {'$gte': min(kpSum)}})
if (kpSum != None): qryList.append({'kpSum': {'$lte': max(kpSum)}})
if (apMean != None): qryList.append({'apMean': {'$gte': min(apMean)}})
if (apMean != None): qryList.append({'apMean': {'$lte': max(apMean)}})
if (sunspot != None): qryList.append({'sunspot': {'$gte': min(sunspot)}})
if (sunspot != None): qryList.append({'sunspot': {'$lte': max(sunspot)}})
#construct the final query definition
qryDict = {'$and': qryList}
#connect to the database
kpData = db.getDataConn(dbName='gme', collName='kp')
#do the query
if (qryList != []): qry = kpData.find(qryDict)
else: qry = kpData.find()
if (qry.count() > 0):
kpList = []
for rec in qry.sort('time'):
kpList.append(kpDay(dbDict=rec))
print '\nreturning a list with', len(kpList), 'days of kp data'
return kpList
#if we didn't find anything ont he mongodb
else:
print '\ncould not find requested data in the mongodb'
print 'we will look on the ftp server, but your conditions will be (mostly) ignored'
if (sTime == None):
print 'start time for search set to 1980...'
sTime = dt.datetime(1980, 1, 1)
kpList = []
if (eTime == None): eTime = dt.now()
for yr in range(sTime.year, eTime.year + 1):
tmpList = readKpFtp(dt.datetime(yr, 1, 1),
eTime=dt.datetime(yr, 12, 31))
if (tmpList == None): continue
for x in tmpList:
kpList.append(x)
if (kpList != []):
print '\nreturning a list with', len(kpList), 'days of kp data'
return kpList
else:
print '\n no data found on FTP server, returning None...'
return None
def mapOmniMongo(sYear,eYear=None,res=5):
"""This function reads omni data from the NASA SPDF FTP server via anonymous FTP connection and maps it to the mongodb.
.. warning::
In general, nobody except the database admins will need to use this function
**Args**:
* **sYear** (int): the year to begin mapping data
* [**eYear**] (int or None): the end year for mapping data. if this is None, eYear will be sYear
* [**res**] (int): the time resolution for mapping data. Can be either 1 or 5. default=5
**Returns**:
* Nothing.
**Example**:
::
gme.ind.mapOmniMongo(1997,res=1)
written by AJ, 20130123
"""
import pydarn.sdio.dbUtils as db
import os, datetime as dt
#check inputs
assert(res == 1 or res == 5),'error, res must be either 1 or 5'
assert(isinstance(sYear,int)),'error, sYear must be int'
if(eYear == None): eYear=sYear
assert(isinstance(eYear,int)),'error, sYear must be None or int'
assert(eYear >= sYear), 'error, end year greater than start year'
#get data connection
mongoData = db.getDataConn(username=os.environ['DBWRITEUSER'],password=os.environ['DBWRITEPASS'],\
dbAddress=os.environ['SDDB'],dbName='gme',collName='omni')
#set up all of the indices
mongoData.ensure_index('time')
mongoData.ensure_index('res')
mongoData.ensure_index('bx')
mongoData.ensure_index('bye')
mongoData.ensure_index('bze')
mongoData.ensure_index('bym')
mongoData.ensure_index('bzm')
mongoData.ensure_index('pDyn')
mongoData.ensure_index('ae')
mongoData.ensure_index('symh')
#read the omni data from the FTP server
for yr in range(sYear,eYear+1):
for mon in range(1,13):
templist = readOmniFtp(dt.datetime(yr,mon,1),dt.datetime(yr,mon,1)+dt.timedelta(days=31),res=res)
if(templist == None): continue
for rec in templist:
#check if a duplicate record exists
qry = mongoData.find({'$and':[{'time':rec.time},{'res':rec.res}]})
print rec.time
tempRec = rec.toDbDict()
cnt = qry.count()
#if this is a new record, insert it
if(cnt == 0): mongoData.insert(tempRec)
#if this is an existing record, update it
elif(cnt == 1):
print 'foundone!!'
dbDict = qry.next()
temp = dbDict['_id']
dbDict = tempRec
dbDict['_id'] = temp
mongoData.save(dbDict)
else:
print 'strange, there is more than 1 record for',rec.time
def readOmni(sTime,eTime=None,res=5,bx=None,bye=None,bze=None,bym=None,bzm=None,pDyn=None,ae=None,symh=None):
"""This function reads omni data. First, it will try to get it from the mongodb, and if it can't find it, it will look on the NASA SPDF FTP server using :func:`readOmniFtp`
**Args**:
* **sTime** (`datetime <http://tinyurl.com/bl352yx>`_ or None): the earliest time you want data for
* [**eTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the latest time you want data for. if this is None, end Time will be 1 day after sTime. default = None
* [**res**] (int): the time reolution of data desired. This can be either 1 or 5. default = 5
* [**bx**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bx values in the range [a,b] will be returned. default = None
* [**bx**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bx values in the range [a,b] will be returned. default = None
* [**bye**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bye values in the range [a,b] will be returned. default = None
* [**bze**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bze values in the range [a,b] will be returned. default = None
* [**bym**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bym values in the range [a,b] will be returned. default = None
* [**bzm**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with bzm values in the range [a,b] will be returned. default = None
* [**pDyn**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with pDyn values in the range [a,b] will be returned. default = None
* [**ae**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with ae values in the range [a,b] will be returned. default = None
* [**symh**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with symh values in the range [a,b] will be returned. default = None
**Returns**:
* **omniList** (list or None): if data is found, a list of :class:`omniRec` objects matching the input parameters is returned. If no data is found, None is returned.
**Example**:
::
import datetime as dt
omniList = gme.ind.readOmni(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),bx=[0,5.5],bye=[-1,3.5],bze=[-10,0],ae=[0,56.3])
written by AJ, 20130128
"""
import datetime as dt
import pydarn.sdio.dbUtils as db
#check all the inputs for validity
assert(isinstance(sTime,dt.datetime)), \
'error, sTime must be a datetime object'
assert(eTime == None or isinstance(eTime,dt.datetime)), \
'error, eTime must be either None or a datetime object'
assert(res==5 or res == 1), 'error, res must be either 1 or 5'
var = locals()
for name in ['bx','bye','bze','bym','bzm','pDyn','ae','symh']:
assert(var[name] == None or (isinstance(var[name],list) and \
isinstance(var[name][0],(int,float)) and isinstance(var[name][1],(int,float)))), \
'error,'+name+' must None or a list of 2 numbers'
if(eTime == None): eTime = sTime+dt.timedelta(days=1)
qryList = []
#if arguments are provided, query for those
qryList.append({'time':{'$gte':sTime}})
qryList.append({'res':res})
if(eTime != None): qryList.append({'time':{'$lte':eTime}})
var = locals()
for name in ['bx','bye','bze','bym','bzm','pDyn','ae','symh']:
if(var[name] != None):
qryList.append({name:{'$gte':min(var[name])}})
qryList.append({name:{'$lte':max(var[name])}})
#construct the final query definition
qryDict = {'$and': qryList}
#connect to the database
omniData = db.getDataConn(dbName='gme',collName='omni')
#do the query
if(qryList != []): qry = omniData.find(qryDict)
else: qry = omniData.find()
if(qry.count() > 0):
omniList = []
for rec in qry.sort('time'):
omniList.append(omniRec(dbDict=rec))
print '\nreturning a list with',len(omniList),'records of omni data'
return omniList
#if we didn't find anything on the mongodb
else:
print '\ncould not find requested data in the mongodb'
print 'we will look on the ftp server, but your conditions will be (mostly) ignored'
#read from ftp server
omniList = readOmniFtp(sTime, eTime)
if(omniList != None):
print '\nreturning a list with',len(omniList),'recs of omni data'
return omniList
else:
print '\n no data found on FTP server, returning None...'
return None
def mapSymAsyMongo(sYear, eYear=None):
"""This function reads sym/asy data from wdc and puts it in mongodb
.. warning::
In general, nobody except the database admins will need to use this function
**Args**:
* **sYear** (int): the year to begin mapping data
* [**eYear**] (int or None): the end year for mapping data. if this is None, eYear will be sYear
**Returns**:
* Nothing.
**Example**:
::
gme.ind.mapSymAsyMongo(2001)
written by AJ, 20130123
"""
import pydarn.sdio.dbUtils as db
import os, datetime as dt
# check inputs
assert isinstance(sYear, int), "error, sYear must be int"
if eYear == None:
eYear = sYear
assert isinstance(eYear, int), "error, sYear must be None or int"
assert eYear >= sYear, "error, end year greater than start year"
# get data connection
mongoData = db.getDataConn(
username=os.environ["DBWRITEUSER"],
password=os.environ["DBWRITEPASS"],
dbAddress=os.environ["SDDB"],
dbName="gme",
collName="symasy",
)
# set up all of the indices
mongoData.ensure_index("time")
mongoData.ensure_index("symh")
mongoData.ensure_index("symd")
mongoData.ensure_index("asyh")
mongoData.ensure_index("asyd")
for yr in range(sYear, eYear + 1):
# 1 day at a time, to not fill up RAM
templist = readSymAsyWeb(dt.datetime(yr, 1, 1), dt.datetime(yr, 1, 1) + dt.timedelta(days=366))
if templist == None:
continue
for rec in templist:
# check if a duplicate record exists
qry = mongoData.find({"time": rec.time})
print rec.time
tempRec = rec.toDbDict()
cnt = qry.count()
# if this is a new record, insert it
if cnt == 0:
mongoData.insert(tempRec)
# if this is an existing record, update it
elif cnt == 1:
print "foundone!!"
dbDict = qry.next()
temp = dbDict["_id"]
dbDict = tempRec
dbDict["_id"] = temp
mongoData.save(dbDict)
else:
print "strange, there is more than 1 Sym/Asy record for", rec.time
del templist
def readSymAsy(sTime=None, eTime=None, symh=None, symd=None, asyh=None, asyd=None):
"""This function reads sym/asy data from the mongodb.
**Args**:
* [**sTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the earliest time you want data for, default=None
* [**eTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the latest time you want data for. if this is None, end Time will be 1 day after sTime. default = None
* [**symh**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with symh values in the range [a,b] will be returned. default = None
* [**symd**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with symd values in the range [a,b] will be returned. default = None
* [**asyh**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with asyh values in the range [a,b] will be returned. default = None
* [**asyd**] (list or None): if this is not None, it must be a 2-element list of numbers, [a,b]. In this case, only data with asyd values in the range [a,b] will be returned. default = None
**Returns**:
* **symList** (list or None): if data is found, a list of :class:`gme.ind.symasy.symAsyRec` objects matching the input parameters is returned. If no data is found, None is returned.
**Example**:
::
import datetime as dt
symList = gme.ind.readSymAsy(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),symh=[5,50],asyd=[-10,0])
written by AJ, 20130131
"""
import datetime as dt
import pydarn.sdio.dbUtils as db
# check all the inputs for validity
assert sTime == None or isinstance(sTime, dt.datetime), "error, sTime must be a datetime object"
assert eTime == None or isinstance(eTime, dt.datetime), "error, eTime must be either None or a datetime object"
var = locals()
for name in ["symh", "symd", "asyh", "asyd"]:
assert var[name] == None or (
isinstance(var[name], list)
and isinstance(var[name][0], (int, float))
and isinstance(var[name][1], (int, float))
), ("error," + name + " must None or a list of 2 numbers")
if eTime == None and sTime != None:
eTime = sTime + dt.timedelta(days=1)
qryList = []
# if arguments are provided, query for those
if sTime != None:
qryList.append({"time": {"$gte": sTime}})
if eTime != None:
qryList.append({"time": {"$lte": eTime}})
var = locals()
for name in ["symh", "symd", "asyh", "asyd"]:
if var[name] != None:
qryList.append({name: {"$gte": min(var[name])}})
qryList.append({name: {"$lte": max(var[name])}})
# construct the final query definition
qryDict = {"$and": qryList}
# connect to the database
symData = db.getDataConn(dbName="gme", collName="symasy")
# do the query
if qryList != []:
qry = symData.find(qryDict)
else:
qry = symData.find()
if qry.count() > 0:
symList = []
for rec in qry.sort("time"):
symList.append(symAsyRec(dbDict=rec))
print "\nreturning a list with", len(symList), "records of sym/asy data"
return symList
# if we didn't find anything on the mongodb
else:
print "\ncould not find requested data in the mongodb"
return None
def readKp(sTime=None,eTime=None,kpMin=None,apMin=None,kpSum=None,apMean=None,sunspot=None):
"""This function reads kp data. First, it will try to get it from the mongodb, and if it can't find it, it will look on the GFZ ftp server using :func:`gme.ind.kp.readKpFtp`
**Args**:
* [**sTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the earliest time you want data for. if this is None, start time will be the earliest record found. default=None
* [**eTime**] (`datetime <http://tinyurl.com/bl352yx>`_ or None): the latest time you want data for. if this is None, end Time will be latest record found. default=None
* [**kpMin**] (int or None): specify this to only return data from dates with a 3-hour kp value of minimum kpMin. if this is none, it will be ignored. default=None
* [**apMin**] (int or None): specify this to only return data from dates with a 3-hour ap value of minimum apMin. if this is none, it will be ignored. default=None
* [**kpSum**] (list or None): this must be a 2 element list of integers. if this is specified, only dates with kpSum values in the range [a,b] will be returned. if this is None, it will be ignored. default=None
* [**apMean**] (list or None): this must be a 2 element list of integers. if this is specified, only dates with apMean values in the range [a,b] will be returned. if this is None, it will be ignored. default=None
* [**sunspot**] (list or None): this must be a 2 element list of integers. if this is specified, only dates with sunspot values in the range [a,b] will be returned. if this is None, it will be ignored. default=None
**Returns**:
* **kpList** (list or None): if data is found, a list of :class:`gme.ind.kp.kpDay` objects matching the input parameters is returned. If not data is found, None is returned.
**Example**:
::
import datetime as dt
kpList = gme.ind.readKp(sTime=dt.datetime(2011,1,1),eTime=dt.datetime(2011,6,1),kpMin=2,apMin=1,kpSum=[0,10],apMean=[0,50],sunspot=[6,100])
written by AJ, 20130123
"""
import datetime as dt
import pydarn.sdio.dbUtils as db
#check all the inputs for validity
assert(sTime == None or isinstance(sTime,dt.datetime)), \
'error, sTime must be either None or a datetime object'
assert(eTime == None or isinstance(eTime,dt.datetime)), \
'error, eTime must be either None or a datetime object'
assert(kpMin == None or isinstance(kpMin,int)), \
'error, kpMin must be either None or an int'
assert(apMin == None or isinstance(apMin,int)), \
'error, apMin must be either None or an int'
assert(kpSum == None or (isinstance(kpSum,list) and len(kpSum) == 2 and \
isinstance(kpSum[0], int) and isinstance(kpSum[1], int))), \
'error, kpSum must be either None or a 2 element list'
assert(apMean == None or (isinstance(apMean,list) and len(apMean) == 2and \
isinstance(apMean[0], int) and isinstance(apMean[1], int))), \
'error, apMean must be either None or a 2 element list'
assert(sunspot == None or (isinstance(sunspot,list) and len(sunspot) == 2and \
isinstance(sunspot[0], int) and isinstance(sunspot[1], int))), \
'error, sunspot must be either None or a 2 element list'
qryList = []
#if arguments are provided, query for those
if(sTime != None): qryList.append({'time':{'$gte':sTime}})
if(eTime != None): qryList.append({'time':{'$lte':eTime}})
if(kpMin != None): qryList.append({'kp':{'$gte':kpMin}})
if(apMin != None): qryList.append({'ap':{'$gte':kpMin}})
if(kpSum != None): qryList.append({'kpSum':{'$gte':min(kpSum)}})
if(kpSum != None): qryList.append({'kpSum':{'$lte':max(kpSum)}})
if(apMean != None): qryList.append({'apMean':{'$gte':min(apMean)}})
if(apMean != None): qryList.append({'apMean':{'$lte':max(apMean)}})
if(sunspot != None): qryList.append({'sunspot':{'$gte':min(sunspot)}})
if(sunspot != None): qryList.append({'sunspot':{'$lte':max(sunspot)}})
#construct the final query definition
qryDict = {'$and': qryList}
#connect to the database
kpData = db.getDataConn(dbName='gme',collName='kp')
#do the query
if(qryList != []): qry = kpData.find(qryDict)
else: qry = kpData.find()
if(qry.count() > 0):
kpList = []
for rec in qry.sort('time'):
kpList.append(kpDay(dbDict=rec))
print '\nreturning a list with',len(kpList),'days of kp data'
return kpList
#if we didn't find anything ont he mongodb
else:
print '\ncould not find requested data in the mongodb'
print 'we will look on the ftp server, but your conditions will be (mostly) ignored'
if(sTime == None):
print 'start time for search set to 1980...'
sTime = dt.datetime(1980,1,1)
kpList = []
if(eTime == None): eTime = dt.now()
for yr in range(sTime.year,eTime.year+1):
tmpList = readKpFtp(dt.datetime(yr,1,1), eTime=dt.datetime(yr,12,31))
if(tmpList == None): continue
for x in tmpList:
kpList.append(x)
if(kpList != []):
print '\nreturning a list with',len(kpList),'days of kp data'
return kpList
else:
print '\n no data found on FTP server, returning None...'
return None
