Example #1
0
def get_rri_ephemeris_full(dat_fname):
    """
    A similar accessor function for RRI ephemeris from the relative path+name 
    of an RRI file, but this time also grabbing the MLT, MLAT, MLONG data.

    *** PARAMS ***
    dat_fname (string): string giving the relative path and filename for the RRI h5 file

    *** RETURNS ***
    glons (float): Geographic Longitude (degrees)
    glats (float): Geographic Latitude (degrees) 
    alts (float): Altitude (mk)
    etimes (float): Ephemeris MET/Truncated JD time (seconds since May 24 1968)
    mlon (float): Magnetic longitude (degrees)
    mlat (float): Magnetic latitude (degrees)
    mlts (float): Magnetic Local Time (hr)
    pitch (float): pitch of CASSIOPE (deg)
    yaw (float): yaw of CASSIOPE (deg)
    roll (float): roll of CASSIOPE (deg)
    """
    import h5py
    from davitpy.models import aacgm
    f = h5py.File(dat_fname)
    geog_longs = f['CASSIOPE Ephemeris']['Geographic Longitude (deg)'].value
    geog_lats  = f['CASSIOPE Ephemeris']['Geographic Latitude (deg)'].value
    ephem_times = f['CASSIOPE Ephemeris']['Ephemeris MET (seconds since May 24, 1968)'].value
    # Wasted processing when other functions call this but don't receive the converted times array
    times = ephems_to_datetime(ephem_times) 
    alts = f['CASSIOPE Ephemeris']['Altitude (km)'].value
    mlat = f['CASSIOPE Ephemeris']['Magnetic Latitude (deg)'].value
    mlon = f['CASSIOPE Ephemeris']['Magnetic Longitude (deg)'].value
    mlts = []
    for i in range(mlon.__len__()):
        dt = times[i]
        lone_mlon = mlon[i]
        mlts.append(aacgm.mltFromYmdhms(dt.year,dt.month,dt.day,dt.hour,dt.minute,dt.second,lone_mlon))
    pitch = f['CASSIOPE Ephemeris']['Pitch (deg)'].value
    yaw = f['CASSIOPE Ephemeris']['Yaw (deg)'].value
    roll = f['CASSIOPE Ephemeris']['Roll (deg)'].value
    return geog_longs,geog_lats,alts,ephem_times,mlon,mlat,mlts,pitch,yaw,roll
Example #2
0
    print "\nComparing magnetic and MLT.  Time selected is " + str(time)
    fig4=plt.figure(4)
    ax=None
    coords="mag"
    tmpmap4 = mapObj(coords=coords, projection="stere", draw=True,
                     boundinglat=40., lat_0=90., lon_0=0., resolution='l',
                     datetime=time, dateTime=time)
    fig5=plt.figure(5)
    ax=None
    coords="mlt"
    tmpmap5 = mapObj(coords=coords, projection="stere", draw=True,
                     boundinglat=40., lat_0=90., lon_0=0., resolution='l',
                     datetime=time, dateTime=time)
    print "MLT at zero MLON should be at " + \
      str(aacgm.mltFromYmdhms(time.year, time.month, time.day,
                              time.hour, time.minute, time.second, 0.))
    print "Figures 4 and 5 should now appear.  Close their windows to continue."
    plt.show()

    print "\nTesting some coordinate transformations."
    print "  Converting geo lat/lon to map x/y to geo lat/lon."
    print "  geo lat/lon to map x/y"

    map1 = mapObj(coords='geo',projection='stere',llcrnrlon=100, llcrnrlat=0, 
                  urcrnrlon=170, urcrnrlat=40, lat_0=54, lon_0=-120,
                  resolution='l', draw=False)
    x,y = map1(-120,54)
    print "    Expected: ",14898932.7446,-14364789.7586
    print "    Received: ",x,y
    print "  map x/y to geo lat/lon"
    lon,lat = map1(x,y,inverse=True,coords='geo')
Example #3
0
                     datetime=time,
                     dateTime=time)
    fig5 = plt.figure(5)
    ax = None
    coords = "mlt"
    tmpmap5 = mapObj(coords=coords,
                     projection="stere",
                     draw=True,
                     boundinglat=40.,
                     lat_0=90.,
                     lon_0=0.,
                     resolution='l',
                     datetime=time,
                     dateTime=time)
    print "MLT at zero MLON should be at " + \
      str(aacgm.mltFromYmdhms(time.year, time.month, time.day,
                              time.hour, time.minute, time.second, 0.))
    print "Figures 4 and 5 should now appear.  Close their windows to continue."
    plt.show()

    print "\nTesting some coordinate transformations."
    print "  Converting geo lat/lon to map x/y to geo lat/lon."
    print "  geo lat/lon to map x/y"

    map1 = mapObj(coords='geo',
                  projection='stere',
                  llcrnrlon=100,
                  llcrnrlat=0,
                  urcrnrlon=170,
                  urcrnrlat=40,
                  lat_0=54,
                  lon_0=-120,
Example #4
0
def coord_conv(lon, lat, start, end, altitude=None, date_time=None,
               end_altitude=None):
    """Convert between geographical, AACGM, and MLT coordinates.  
    date_time must be set to use any AACGM systems.
  
    Parameters
    ----------
    lon : float
        longitude (MLT must be in degrees, not hours)
    lat : float
        latitude
    start : str
        coordinate system of input. Options: 'geo', 'mag', 'mlt'
    end : str
        desired output coordinate system. Options: 'geo', 'mag', 'mlt'
    altitude : Optional[int/float/list]
        altitude to be used (km).  Can be int/float or
        list of same size as lon and lat.  Default:  None
    date_time : Optional[datetime]
        Default:  None
    end_altitude : Optional[int/float/list]
        used for conversions from coords at one
        altitude to coords at another.  In km.  Can be int/float or
        list of same size as lon and lat.  Default:  None

    Returns
    -------
    lon : (float, list, or numpy array) 
        MLT is in degrees, not hours.  Output type is the same as
        input type (except int becomes float)
    lat : (float, list, numpy array)
        MLT is in degrees, not hours.  Output type is the same as
        input type (except int becomes float)

    Example
    -------
        import utils
        lon, lat = utils.coord_conv(lon, lat, 'geo', 'mlt',
                                    altitude=300.,
                                    date_time=datetime(2012,3,12,0,56))
        
    Notes
    -----
    A how-to for expansion of this function to handle new coordinate
    systems is included in the code comments.

    original version written by Matt W., 2013-09, based on code by...Sebastien?
    brand new version by Matt W., 2014-08

    """
    import numpy as np
    
    from davitpy.models import aacgm
    from davitpy.utils.coordUtils import get_coord_dict

    ####################################################################
    #                                                                  #
    # Sections of code that must be modified to add new coordinate     #
    # systems are highlighted by pound-lines (like this comment is)    #
    # and are provided with instructions in the comments.              #
    #                                                                  #
    ####################################################################

    ####################################################################
    # Coordinate systems are named and listed in this block. Add a new 
    # system to coords_dict with a code (for start and end) and a name.
    # Add the code to the list for the family it belongs to, or create a
    # new list if adding a new family.  Finally, add the code to the 
    # appropriate list if the system requires altitude or date_time.

    # Define acceptable coordinate systems in the function 
    # get_coord_dict
    
    # List all systems in the AACGM family.
    aacgm_sys = ["mag", "mlt"]

    # List all systems that require altitude.
    alti_sys = ["mag", "mlt"]

    # List all systems that require date_time.
    dt_sys = ["mag", "mlt"]

    # End of system list block.
    ####################################################################

    coords_dict, coords_string = get_coord_dict()

    # Create a string for printing of systems requiring altitude.
    alti_string = ""
    for code in alti_sys:
        alti_string += "\n" + coords_dict[code] + " (" + code + ")"

    # Create a string for printing of systems requiring datetime.
    dt_string = ""
    for code in dt_sys:
        dt_string += "\n" + coords_dict[code] + " (" + code + ")"

    # Check that the coordinates are possible.
    assert(start in coords_dict and end in coords_dict),\
            logging.error("Start coords are " + start +
                          " and end coords are " + end +
                          ".\n" + coords_string)

    # Check whether altitude is needed and provided.
    if start in alti_sys or end in alti_sys or end_altitude is not None:
        assert(altitude is not None),\
                logging.error("altitude must be provided for: " +
                              alti_string + "\nto perform altitude " +
                              "conversions")
    
    # Check whether date_time is needed and provided.
    if start in dt_sys or end in dt_sys:
        assert(date_time is not None),\
                logging.error("date_time must be provided for: " +
                              dt_string)

    # Sanitise inputs.
    if isinstance(lon, int):
        lon, lat = float(lon), float(lat)
    is_list = isinstance(lon, (list, tuple))
    is_float = isinstance(lon, float)
    if is_float:
        lon, lat = [lon], [lat]
    if not (is_float or is_list):
        assert(isinstance(lon, np.ndarray)),\
                logging.error("Must input int, float, list, or " +
                              "numpy array.")

    # Make the inputs into numpy arrays because single element lists 
    # have no len.
    lon, lat = np.array(lon), np.array(lat)
    orig_shape = np.shape(lon)
    lon, lat = lon.flatten(), lat.flatten()

    # Test whether we are using the same altitude for everything.
    if altitude is not None:
        alt = np.array(altitude)
        if np.size(alt) == 1:
            altitude = [altitude]*np.size(lon)

    if end_altitude is not None:
        e_alt = np.array(end_altitude)
        if np.size(e_alt) == 1:
            end_altitude = [end_altitude]*np.size(lon)

    # Set a flag that we are doing and altitude conversion.
    alt_conv = (end_altitude is not None and end_altitude != altitude)

    ####################################################################
    # FROM conversions for system families are performed in this 
    # section.  Within the family of the start system, convert into the 
    # base system for that family.  If the end system is not in the same
    # family, end by converting from the base system into geographic.
    #
    # Add new systems and families by following the example of the AACGM 
    # family block.

    # Check whether there is a conversion to do.
    if start != end or alt_conv:

        ################################################################
        # AACGM family FROM conversions.
        # This is the reason for having the aacgm_sys list:
        if start in aacgm_sys:

            # Convert all other AACGM systems to AACGM.  Follow the
            # example of the MLT block to add new systems within the
            # family.

            ############################################################
            # Convert MLT to AACGM
            if start == "mlt":
                # Convert MLT from degrees to hours.
                lon *= 24./360.
                # Sanitise for later.
                lon %= 24.
                # Find MLT of 0 magnetic lon.
                mlt_0 = aacgm.mltFromYmdhms(date_time.year, date_time.month,
                                            date_time.day, date_time.hour,
                                            date_time.minute, date_time.second,
                                            0.)     
                # Calculate MLT difference, which is magnetic lon in hours.
                lon -= mlt_0
                # Sanitise and convert to degrees.
                lon %= 24.
                lon *= 360./24.
                # Covert from (0,360) to (-180,180).
                lon[np.where(lon > 180.)] -= 360.
                start = "mag"

            # End of MLT FROM block.
            ############################################################
        
            # Now it is in AACGM.  
            assert(start == "mag"),logging.error("should be in AACGM now")

            # If the end result is not an AACGM system or there is an
            # altitude conversion, convert to geo.
            if (end not in aacgm_sys) or alt_conv:
                lat, lon, _ = aacgm.aacgmConvArr(list(lat), list(lon), 
                                                 altitude, date_time.year, 1)
                lon, lat = np.array(lon), np.array(lat)
                start = "geo"
        
        # End of AACGM family FROM block.
        ################################################################

    # End of FROM block.
    ####################################################################

    # Now it is in:
    # AACGM if the start and end are in an AACGM system
    # geo otherwise
    # Add to this list for new system families to help keep track.

    # If there is an altitude conversion, it will be in geo now and the
    # next step is to convert to the end system even if it's the same
    # as the start system.  When we do that we want to set:
    if alt_conv:
        altitude = end_altitude

    ####################################################################
    # TO conversions for system families are performed in this 
    # section.  If the start system was not in the same family as the
    # end system (i.e., it is now in geographic), convert into the base 
    # system for the family of the end system.  Then convert within the
    # family to the end system.
    #
    # Add new systems and families by following the example of the AACGM 
    # family block.
    
    # Check whether there is still a conversion to do.  If the
    # conversion is to geographic or to the base system of whatever
    # family the start was in, then all the work is done.
    if start != end:

        ################################################################
        # AACGM family TO conversions.
        if end in aacgm_sys:
            # If it isn't in AACGM already it's in geo.
            if start == "geo":
                lat, lon, _ = aacgm.aacgmConvArr(list(lat), list(lon), 
                                                 altitude, date_time.year, 0)
                lon, lat = np.array(lon), np.array(lat)
                start = "mag"

            # It is in AACGM now.
            assert(start == "mag"),logging.error("should be in AACGM now")

            # Convert AACGM to all other AACGM systems.  Follow the
            # example of the MLT block to add new systems within the
            # family.

            ############################################################
            # MLT TO conversions.
            if end == "mlt":
                for num, el in enumerate(lon):
                    # Find MLT from magnetic lon and datetime.
                    lon[num] = aacgm.mltFromYmdhms(date_time.year, 
                                                   date_time.month, 
                                                   date_time.day, 
                                                   date_time.hour, 
                                                   date_time.minute, 
                                                   date_time.second, 
                                                   el)
                # Convert hours to degrees.
                lon *= 360./24.
                # Convert from (0,360) to (-180,180).
                lon[np.where(lon > 180.)] -= 360.
                start = "mlt"
            
            # End of MLT TO block.
            ############################################################

        # End of AACGM family TO block.
        ################################################################

    # End of TO block.
    ####################################################################

    # Now it should be in the end system.
    assert(start == end),logging.error("not in correct end system...?????")

    # Convert outputs to input type and shape.
    if is_list:
        lon, lat = list(lon), list(lat)
    elif is_float:
        lon, lat = list(lon)[0], list(lat)[0]
    else:
        # Otherwise it stays a numpy array.
        lon, lat = lon.reshape(orig_shape), lat.reshape(orig_shape)

    return lon, lat
Example #5
0
class FitESUtils(object):
    """
    A class to read fitacf data records
    and write them to es
    """
    def __init__(self):
        import es_utils
        # set up connections
        self.esU = es_utils.ElasticUtils({"host" : "128.173.145.158",\
             "port" : 9200})
        self.fitIndName = "data-superdarn"
        self.fitTypeName = 'fit_data'
        # Once we cross the limit insert the recs in to es
        self.insertRecLimit = 100000

    def create_fit_index(self):
        # create fit data index for es
        reqBody = {
            "settings": {
                "number_of_shards": 1,
                "number_of_replicas": 0
            },
            'mappings': {
                self.fitTypeName: {
                    'properties': {
                        'gates': {
                            'type': 'integer'
                        },
                        'ranges': {
                            'type': 'integer'
                        },
                        'pwr_0': {
                            'type': 'float'
                        },
                        'pwr_l': {
                            'type': 'float'
                        },
                        'vel': {
                            'type': 'float'
                        },
                        'gsct': {
                            'type': 'int'
                        },
                        'vel_err': {
                            'type': 'float'
                        },
                        'width': {
                            'type': 'float'
                        },
                        'glat': {
                            'type': 'float'
                        },
                        'glon': {
                            'type': 'float'
                        },
                        'gazm': {
                            'type': 'float'
                        },
                        'mlat': {
                            'type': 'float'
                        },
                        'mlon': {
                            'type': 'float'
                        },
                        'mlt': {
                            'type': 'float'
                        },
                        'mazm': {
                            'type': 'float'
                        },
                        'ts': {
                            'type': 'date',
                            'format': 'dateOptionalTime'
                        },
                        'radname': {
                            'type': 'keyword'
                        },
                        'filetype': {
                            'type': 'keyword'
                        },
                        'bmnum': {
                            'type': 'integer'
                        },
                        'tfreq': {
                            'type': 'integer'
                        },
                        'scanflg': {
                            'type': 'keyword'
                        },
                        'npnts': {
                            'type': 'integer'
                        },
                        'nrang': {
                            'type': 'integer'
                        },
                        'channel': {
                            'type': 'keyword'
                        },
                        'cpid': {
                            'type': 'integer'
                        },
                    }
                }
            }
        }
        self.esU.createIndex( self.fitIndName, \
            requestBody=reqBody, deleteOld=True )
        print "created index--->", self.fitIndName

    def insert_fit_records(self, fileName):
        # given a fit record file read its contents
        # this function is mostly taken/copied from DaVitPy.
        # open the file
        import datetime
        from davitpy.models import aacgm
        try:
            fp = open(fileName)
        except Exception, e:
            print e
            print 'problem opening the file %s', fileName
            return None

        # read the first line
        line = fp.readline()
        cnt = 1
        totRecCnt = 0
        fitRecs = []
        while line:
            # split the lines by whitespace
            cols = line.split()
            # check for first header line
            if cnt == 1:
                d, t = cols[0], cols[1]
                # parse the line into a datetime object
                currDt = datetime.datetime(int(d[:4]), int(d[5:7]), int(d[8:]),
                                           int(t[:2]), int(t[3:5]), int(t[6:]))
                radname = cols[2]
                filetype = cols[3]
            # check for second header line
            elif cnt == 2:
                bmnum = int(cols[2])
                tfreq = int(cols[5])
                scanflg = cols[17]
            # check for third header line
            elif cnt == 3:
                npnts = int(cols[2])
                nrang = int(cols[5])
                channel = cols[8]
                cpid = int(cols[11])
            # check for fourth header line
            elif cnt == 4:
                # empty lists to hold the fitted parameters
                gates, ranges, pwr_0, pwr_l, vel, gsct, vel_err, width = [], [], [], [], [], [], [], []
                glat, glon, gazm, mlat, mlon, mazm = [], [], [], [], [], []
                # Store the fit recs in an array to ingest them to es later
                # read all of the reange gates
                for i in range(npnts):
                    line = fp.readline()
                    cols = line.split()
                    # Enter the values in a dict
                    fitDict = {}
                    fitDict["gates"] = int(cols[0])
                    fitDict["ranges"] = int(cols[1])
                    fitDict["pwr_0"] = float(cols[2])
                    fitDict["pwr_l"] = float(cols[4])
                    fitDict["vel"] = float(cols[5])
                    fitDict["gsct"] = int(cols[6])
                    fitDict["vel_err"] = float(cols[7])
                    fitDict["width"] = float(cols[8])
                    fitDict["glat"] = float(cols[9])
                    fitDict["glon"] = float(cols[10])
                    fitDict["gazm"] = float(cols[11])
                    fitDict["mlat"] = float(cols[12])
                    fitDict["mlon"] = float(cols[13])
                    fitDict["mazm"] = float(cols[14])
                    fitDict["ts"] = currDt.strftime("%Y-%m-%dT%H:%M:%S")
                    fitDict["radname"] = radname
                    fitDict["filetype"] = filetype
                    fitDict["bmnum"] = bmnum
                    fitDict["tfreq"] = tfreq
                    fitDict["scanflg"] = scanflg
                    fitDict["npnts"] = npnts
                    fitDict["nrang"] = nrang
                    fitDict["channel"] = channel
                    fitDict["cpid"] = cpid
                    # get the mlt value
                    calcMlt = aacgm.mltFromYmdhms(currDt.year, \
                        currDt.month,currDt.day, currDt.hour,\
                        currDt.minute, currDt.second, fitDict["mlon"])
                    fitDict["mlt"] = round(calcMlt, 2)
                    # store in the array
                    fit_op_dict = {
                        "index": {
                            "_index": self.fitIndName,
                            "_type": self.fitTypeName
                        }
                    }
                    fitRecs.append(fit_op_dict)
                    fitRecs.append(fitDict)
                    totRecCnt += 1
                # read the blank line after each record
                line = fp.readline()
                # reset the count
                cnt = 0
                if cnt == 0:
                    # if there are too many records
                    # insert them into es and read fresh one's
                    if len(fitRecs) > self.insertRecLimit:
                        self.esU.insert_data_recs(self.fitIndName, fitRecs)
                        print "---dumped " + str(
                            totRecCnt) + " records into es---"
                        fitRecs = []
                        print "reading fresh records!"
            # read the next line
            line = fp.readline()
            cnt += 1
        print "---dumping final " + str(len(fitRecs)) + " records into es---"
        self.esU.insert_data_recs(self.fitIndName, fitRecs)