def get_rri_ephemeris_full(dat_fname): """ A similar accessor function for RRI ephemeris from the relative path+name of an RRI file, but this time also grabbing the MLT, MLAT, MLONG data. *** PARAMS *** dat_fname (string): string giving the relative path and filename for the RRI h5 file *** RETURNS *** glons (float): Geographic Longitude (degrees) glats (float): Geographic Latitude (degrees) alts (float): Altitude (mk) etimes (float): Ephemeris MET/Truncated JD time (seconds since May 24 1968) mlon (float): Magnetic longitude (degrees) mlat (float): Magnetic latitude (degrees) mlts (float): Magnetic Local Time (hr) pitch (float): pitch of CASSIOPE (deg) yaw (float): yaw of CASSIOPE (deg) roll (float): roll of CASSIOPE (deg) """ import h5py from davitpy.models import aacgm f = h5py.File(dat_fname) geog_longs = f['CASSIOPE Ephemeris']['Geographic Longitude (deg)'].value geog_lats = f['CASSIOPE Ephemeris']['Geographic Latitude (deg)'].value ephem_times = f['CASSIOPE Ephemeris']['Ephemeris MET (seconds since May 24, 1968)'].value # Wasted processing when other functions call this but don't receive the converted times array times = ephems_to_datetime(ephem_times) alts = f['CASSIOPE Ephemeris']['Altitude (km)'].value mlat = f['CASSIOPE Ephemeris']['Magnetic Latitude (deg)'].value mlon = f['CASSIOPE Ephemeris']['Magnetic Longitude (deg)'].value mlts = [] for i in range(mlon.__len__()): dt = times[i] lone_mlon = mlon[i] mlts.append(aacgm.mltFromYmdhms(dt.year,dt.month,dt.day,dt.hour,dt.minute,dt.second,lone_mlon)) pitch = f['CASSIOPE Ephemeris']['Pitch (deg)'].value yaw = f['CASSIOPE Ephemeris']['Yaw (deg)'].value roll = f['CASSIOPE Ephemeris']['Roll (deg)'].value return geog_longs,geog_lats,alts,ephem_times,mlon,mlat,mlts,pitch,yaw,roll
print "\nComparing magnetic and MLT. Time selected is " + str(time) fig4=plt.figure(4) ax=None coords="mag" tmpmap4 = mapObj(coords=coords, projection="stere", draw=True, boundinglat=40., lat_0=90., lon_0=0., resolution='l', datetime=time, dateTime=time) fig5=plt.figure(5) ax=None coords="mlt" tmpmap5 = mapObj(coords=coords, projection="stere", draw=True, boundinglat=40., lat_0=90., lon_0=0., resolution='l', datetime=time, dateTime=time) print "MLT at zero MLON should be at " + \ str(aacgm.mltFromYmdhms(time.year, time.month, time.day, time.hour, time.minute, time.second, 0.)) print "Figures 4 and 5 should now appear. Close their windows to continue." plt.show() print "\nTesting some coordinate transformations." print " Converting geo lat/lon to map x/y to geo lat/lon." print " geo lat/lon to map x/y" map1 = mapObj(coords='geo',projection='stere',llcrnrlon=100, llcrnrlat=0, urcrnrlon=170, urcrnrlat=40, lat_0=54, lon_0=-120, resolution='l', draw=False) x,y = map1(-120,54) print " Expected: ",14898932.7446,-14364789.7586 print " Received: ",x,y print " map x/y to geo lat/lon" lon,lat = map1(x,y,inverse=True,coords='geo')
datetime=time, dateTime=time) fig5 = plt.figure(5) ax = None coords = "mlt" tmpmap5 = mapObj(coords=coords, projection="stere", draw=True, boundinglat=40., lat_0=90., lon_0=0., resolution='l', datetime=time, dateTime=time) print "MLT at zero MLON should be at " + \ str(aacgm.mltFromYmdhms(time.year, time.month, time.day, time.hour, time.minute, time.second, 0.)) print "Figures 4 and 5 should now appear. Close their windows to continue." plt.show() print "\nTesting some coordinate transformations." print " Converting geo lat/lon to map x/y to geo lat/lon." print " geo lat/lon to map x/y" map1 = mapObj(coords='geo', projection='stere', llcrnrlon=100, llcrnrlat=0, urcrnrlon=170, urcrnrlat=40, lat_0=54, lon_0=-120,
def coord_conv(lon, lat, start, end, altitude=None, date_time=None, end_altitude=None): """Convert between geographical, AACGM, and MLT coordinates. date_time must be set to use any AACGM systems. Parameters ---------- lon : float longitude (MLT must be in degrees, not hours) lat : float latitude start : str coordinate system of input. Options: 'geo', 'mag', 'mlt' end : str desired output coordinate system. Options: 'geo', 'mag', 'mlt' altitude : Optional[int/float/list] altitude to be used (km). Can be int/float or list of same size as lon and lat. Default: None date_time : Optional[datetime] Default: None end_altitude : Optional[int/float/list] used for conversions from coords at one altitude to coords at another. In km. Can be int/float or list of same size as lon and lat. Default: None Returns ------- lon : (float, list, or numpy array) MLT is in degrees, not hours. Output type is the same as input type (except int becomes float) lat : (float, list, numpy array) MLT is in degrees, not hours. Output type is the same as input type (except int becomes float) Example ------- import utils lon, lat = utils.coord_conv(lon, lat, 'geo', 'mlt', altitude=300., date_time=datetime(2012,3,12,0,56)) Notes ----- A how-to for expansion of this function to handle new coordinate systems is included in the code comments. original version written by Matt W., 2013-09, based on code by...Sebastien? brand new version by Matt W., 2014-08 """ import numpy as np from davitpy.models import aacgm from davitpy.utils.coordUtils import get_coord_dict #################################################################### # # # Sections of code that must be modified to add new coordinate # # systems are highlighted by pound-lines (like this comment is) # # and are provided with instructions in the comments. # # # #################################################################### #################################################################### # Coordinate systems are named and listed in this block. Add a new # system to coords_dict with a code (for start and end) and a name. # Add the code to the list for the family it belongs to, or create a # new list if adding a new family. Finally, add the code to the # appropriate list if the system requires altitude or date_time. # Define acceptable coordinate systems in the function # get_coord_dict # List all systems in the AACGM family. aacgm_sys = ["mag", "mlt"] # List all systems that require altitude. alti_sys = ["mag", "mlt"] # List all systems that require date_time. dt_sys = ["mag", "mlt"] # End of system list block. #################################################################### coords_dict, coords_string = get_coord_dict() # Create a string for printing of systems requiring altitude. alti_string = "" for code in alti_sys: alti_string += "\n" + coords_dict[code] + " (" + code + ")" # Create a string for printing of systems requiring datetime. dt_string = "" for code in dt_sys: dt_string += "\n" + coords_dict[code] + " (" + code + ")" # Check that the coordinates are possible. assert(start in coords_dict and end in coords_dict),\ logging.error("Start coords are " + start + " and end coords are " + end + ".\n" + coords_string) # Check whether altitude is needed and provided. if start in alti_sys or end in alti_sys or end_altitude is not None: assert(altitude is not None),\ logging.error("altitude must be provided for: " + alti_string + "\nto perform altitude " + "conversions") # Check whether date_time is needed and provided. if start in dt_sys or end in dt_sys: assert(date_time is not None),\ logging.error("date_time must be provided for: " + dt_string) # Sanitise inputs. if isinstance(lon, int): lon, lat = float(lon), float(lat) is_list = isinstance(lon, (list, tuple)) is_float = isinstance(lon, float) if is_float: lon, lat = [lon], [lat] if not (is_float or is_list): assert(isinstance(lon, np.ndarray)),\ logging.error("Must input int, float, list, or " + "numpy array.") # Make the inputs into numpy arrays because single element lists # have no len. lon, lat = np.array(lon), np.array(lat) orig_shape = np.shape(lon) lon, lat = lon.flatten(), lat.flatten() # Test whether we are using the same altitude for everything. if altitude is not None: alt = np.array(altitude) if np.size(alt) == 1: altitude = [altitude]*np.size(lon) if end_altitude is not None: e_alt = np.array(end_altitude) if np.size(e_alt) == 1: end_altitude = [end_altitude]*np.size(lon) # Set a flag that we are doing and altitude conversion. alt_conv = (end_altitude is not None and end_altitude != altitude) #################################################################### # FROM conversions for system families are performed in this # section. Within the family of the start system, convert into the # base system for that family. If the end system is not in the same # family, end by converting from the base system into geographic. # # Add new systems and families by following the example of the AACGM # family block. # Check whether there is a conversion to do. if start != end or alt_conv: ################################################################ # AACGM family FROM conversions. # This is the reason for having the aacgm_sys list: if start in aacgm_sys: # Convert all other AACGM systems to AACGM. Follow the # example of the MLT block to add new systems within the # family. ############################################################ # Convert MLT to AACGM if start == "mlt": # Convert MLT from degrees to hours. lon *= 24./360. # Sanitise for later. lon %= 24. # Find MLT of 0 magnetic lon. mlt_0 = aacgm.mltFromYmdhms(date_time.year, date_time.month, date_time.day, date_time.hour, date_time.minute, date_time.second, 0.) # Calculate MLT difference, which is magnetic lon in hours. lon -= mlt_0 # Sanitise and convert to degrees. lon %= 24. lon *= 360./24. # Covert from (0,360) to (-180,180). lon[np.where(lon > 180.)] -= 360. start = "mag" # End of MLT FROM block. ############################################################ # Now it is in AACGM. assert(start == "mag"),logging.error("should be in AACGM now") # If the end result is not an AACGM system or there is an # altitude conversion, convert to geo. if (end not in aacgm_sys) or alt_conv: lat, lon, _ = aacgm.aacgmConvArr(list(lat), list(lon), altitude, date_time.year, 1) lon, lat = np.array(lon), np.array(lat) start = "geo" # End of AACGM family FROM block. ################################################################ # End of FROM block. #################################################################### # Now it is in: # AACGM if the start and end are in an AACGM system # geo otherwise # Add to this list for new system families to help keep track. # If there is an altitude conversion, it will be in geo now and the # next step is to convert to the end system even if it's the same # as the start system. When we do that we want to set: if alt_conv: altitude = end_altitude #################################################################### # TO conversions for system families are performed in this # section. If the start system was not in the same family as the # end system (i.e., it is now in geographic), convert into the base # system for the family of the end system. Then convert within the # family to the end system. # # Add new systems and families by following the example of the AACGM # family block. # Check whether there is still a conversion to do. If the # conversion is to geographic or to the base system of whatever # family the start was in, then all the work is done. if start != end: ################################################################ # AACGM family TO conversions. if end in aacgm_sys: # If it isn't in AACGM already it's in geo. if start == "geo": lat, lon, _ = aacgm.aacgmConvArr(list(lat), list(lon), altitude, date_time.year, 0) lon, lat = np.array(lon), np.array(lat) start = "mag" # It is in AACGM now. assert(start == "mag"),logging.error("should be in AACGM now") # Convert AACGM to all other AACGM systems. Follow the # example of the MLT block to add new systems within the # family. ############################################################ # MLT TO conversions. if end == "mlt": for num, el in enumerate(lon): # Find MLT from magnetic lon and datetime. lon[num] = aacgm.mltFromYmdhms(date_time.year, date_time.month, date_time.day, date_time.hour, date_time.minute, date_time.second, el) # Convert hours to degrees. lon *= 360./24. # Convert from (0,360) to (-180,180). lon[np.where(lon > 180.)] -= 360. start = "mlt" # End of MLT TO block. ############################################################ # End of AACGM family TO block. ################################################################ # End of TO block. #################################################################### # Now it should be in the end system. assert(start == end),logging.error("not in correct end system...?????") # Convert outputs to input type and shape. if is_list: lon, lat = list(lon), list(lat) elif is_float: lon, lat = list(lon)[0], list(lat)[0] else: # Otherwise it stays a numpy array. lon, lat = lon.reshape(orig_shape), lat.reshape(orig_shape) return lon, lat
class FitESUtils(object): """ A class to read fitacf data records and write them to es """ def __init__(self): import es_utils # set up connections self.esU = es_utils.ElasticUtils({"host" : "128.173.145.158",\ "port" : 9200}) self.fitIndName = "data-superdarn" self.fitTypeName = 'fit_data' # Once we cross the limit insert the recs in to es self.insertRecLimit = 100000 def create_fit_index(self): # create fit data index for es reqBody = { "settings": { "number_of_shards": 1, "number_of_replicas": 0 }, 'mappings': { self.fitTypeName: { 'properties': { 'gates': { 'type': 'integer' }, 'ranges': { 'type': 'integer' }, 'pwr_0': { 'type': 'float' }, 'pwr_l': { 'type': 'float' }, 'vel': { 'type': 'float' }, 'gsct': { 'type': 'int' }, 'vel_err': { 'type': 'float' }, 'width': { 'type': 'float' }, 'glat': { 'type': 'float' }, 'glon': { 'type': 'float' }, 'gazm': { 'type': 'float' }, 'mlat': { 'type': 'float' }, 'mlon': { 'type': 'float' }, 'mlt': { 'type': 'float' }, 'mazm': { 'type': 'float' }, 'ts': { 'type': 'date', 'format': 'dateOptionalTime' }, 'radname': { 'type': 'keyword' }, 'filetype': { 'type': 'keyword' }, 'bmnum': { 'type': 'integer' }, 'tfreq': { 'type': 'integer' }, 'scanflg': { 'type': 'keyword' }, 'npnts': { 'type': 'integer' }, 'nrang': { 'type': 'integer' }, 'channel': { 'type': 'keyword' }, 'cpid': { 'type': 'integer' }, } } } } self.esU.createIndex( self.fitIndName, \ requestBody=reqBody, deleteOld=True ) print "created index--->", self.fitIndName def insert_fit_records(self, fileName): # given a fit record file read its contents # this function is mostly taken/copied from DaVitPy. # open the file import datetime from davitpy.models import aacgm try: fp = open(fileName) except Exception, e: print e print 'problem opening the file %s', fileName return None # read the first line line = fp.readline() cnt = 1 totRecCnt = 0 fitRecs = [] while line: # split the lines by whitespace cols = line.split() # check for first header line if cnt == 1: d, t = cols[0], cols[1] # parse the line into a datetime object currDt = datetime.datetime(int(d[:4]), int(d[5:7]), int(d[8:]), int(t[:2]), int(t[3:5]), int(t[6:])) radname = cols[2] filetype = cols[3] # check for second header line elif cnt == 2: bmnum = int(cols[2]) tfreq = int(cols[5]) scanflg = cols[17] # check for third header line elif cnt == 3: npnts = int(cols[2]) nrang = int(cols[5]) channel = cols[8] cpid = int(cols[11]) # check for fourth header line elif cnt == 4: # empty lists to hold the fitted parameters gates, ranges, pwr_0, pwr_l, vel, gsct, vel_err, width = [], [], [], [], [], [], [], [] glat, glon, gazm, mlat, mlon, mazm = [], [], [], [], [], [] # Store the fit recs in an array to ingest them to es later # read all of the reange gates for i in range(npnts): line = fp.readline() cols = line.split() # Enter the values in a dict fitDict = {} fitDict["gates"] = int(cols[0]) fitDict["ranges"] = int(cols[1]) fitDict["pwr_0"] = float(cols[2]) fitDict["pwr_l"] = float(cols[4]) fitDict["vel"] = float(cols[5]) fitDict["gsct"] = int(cols[6]) fitDict["vel_err"] = float(cols[7]) fitDict["width"] = float(cols[8]) fitDict["glat"] = float(cols[9]) fitDict["glon"] = float(cols[10]) fitDict["gazm"] = float(cols[11]) fitDict["mlat"] = float(cols[12]) fitDict["mlon"] = float(cols[13]) fitDict["mazm"] = float(cols[14]) fitDict["ts"] = currDt.strftime("%Y-%m-%dT%H:%M:%S") fitDict["radname"] = radname fitDict["filetype"] = filetype fitDict["bmnum"] = bmnum fitDict["tfreq"] = tfreq fitDict["scanflg"] = scanflg fitDict["npnts"] = npnts fitDict["nrang"] = nrang fitDict["channel"] = channel fitDict["cpid"] = cpid # get the mlt value calcMlt = aacgm.mltFromYmdhms(currDt.year, \ currDt.month,currDt.day, currDt.hour,\ currDt.minute, currDt.second, fitDict["mlon"]) fitDict["mlt"] = round(calcMlt, 2) # store in the array fit_op_dict = { "index": { "_index": self.fitIndName, "_type": self.fitTypeName } } fitRecs.append(fit_op_dict) fitRecs.append(fitDict) totRecCnt += 1 # read the blank line after each record line = fp.readline() # reset the count cnt = 0 if cnt == 0: # if there are too many records # insert them into es and read fresh one's if len(fitRecs) > self.insertRecLimit: self.esU.insert_data_recs(self.fitIndName, fitRecs) print "---dumped " + str( totRecCnt) + " records into es---" fitRecs = [] print "reading fresh records!" # read the next line line = fp.readline() cnt += 1 print "---dumping final " + str(len(fitRecs)) + " records into es---" self.esU.insert_data_recs(self.fitIndName, fitRecs)