def read_records(self) -> collections.deque: """ This method reads the records from the /stream passed into the instance. Return ------- _dmap_records : collections.Deque Deque list of DmapRecords (ordered dictionary) See Also -------- DmapScalar : DMap record's scalar data structure DmapArray : DMap record's array data structure See DEVELOPER_README.md for more information on Dmap data structure. """ # read bytes until end of byte array pydarn_log.debug("Reading DMap records") while self.cursor < self.dmap_end_bytes: new_record = self.read_record() self._dmap_records.append(new_record) self.rec_num += 1 self.bytes_check(self.cursor, "cursor", self.dmap_end_bytes, "total bytes in the file") self._records = dmap2dict(self._dmap_records) return self._records
def read_map(self) -> List[dict]: """ Reads Map DMAP file/stream Returns ------- dmap_records : List[dict] DMAP record of the Map data """ pydarn_log.debug("Reading Map file: {}".format(self.dmap_file)) # We need to separate the fields into subsets because map files # can exclude extra fields if the grid file does not contain them. # Other subsets are related to what processing commands are used to # generate the final map file, example: map_addhmb. This command is # not necessarily to generate a map file but it add the Heppner Maynard # boundary to the map file. See missing_field_check # method in SDarnUtilities for more information. file_struct_list = [ superdarn_formats.Map.types, superdarn_formats.Map.extra_fields, superdarn_formats.Map.fit_fields, superdarn_formats.Map.hmb_fields, superdarn_formats.Map.model_fields ] self._read_darn_records(file_struct_list) self.records = dmap2dict(self._dmap_records) return self.records
def test_DmapRead_dmap2dict_dict2dmap(self): """ Test DmapRead from a file and convert to a dictionary. """ dmap_read = pydarn.DmapRead(rawacf_file) records = dmap_read.read_records() records = dmap_read.get_dmap_records dict_records = pydarn.dmap2dict(records) records_2 = pydarn.dict2dmap(dict_records) self.dmap_compare(records, records_2)
def test_dmap2dict(self): """ From utils package, testing dmap2dict function """ # need to break up the list of dictionaries to properly # compare each field value dmap_list_test = pydarn.dmap2dict(self.dmap_records) for j in range(len(dmap_list_test)): for key, value in dmap_list_test[j].items(): if isinstance(value, np.ndarray): self.assertTrue(np.array_equal(value, self.dmap_list[j][key])) else: self.assertEqual(value, self.dmap_list[j][key])
def read_iqdat(self) -> List[dict]: """ Reads iqdat DMAP file/stream Returns ------- dmap_records : List[dict] DMAP record of the Iqdat data """ pydarn_log.debug("Reading Iqdat file: {}".format(self.dmap_file)) file_struct_list = [superdarn_formats.Iqdat.types] self._read_darn_records(file_struct_list) self.records = dmap2dict(self._dmap_records) return self.records
def read_rawacf(self) -> List[dict]: """ Reads Rawacf DMAP file/stream Returns ------- dmap_records : List[dict] DMAP record of the Rawacf data """ pydarn_log.debug("Reading Rawacf file: {}".format(self.dmap_file)) file_struct_list = [ superdarn_formats.Rawacf.types, superdarn_formats.Rawacf.extra_fields, superdarn_formats.Rawacf.cross_correlation_field ] self._read_darn_records(file_struct_list) self.records = dmap2dict(self._dmap_records) return self.records
def read_grid(self) -> List[dict]: """ Reads Grid DMAP file/stream Returns ------- dmap_records : List[dict] DMAP record of the Grid data """ pydarn_log.debug("Reading Grid file: {}".format(self.dmap_file)) # We need to separate the fields into subsets because grid files # can exclude extra fields if the option -ext is not passed in # when generating the grid file in RST. See missing_field_check # method in SDarnUtilities for more information. file_struct_list = [ superdarn_formats.Grid.types, superdarn_formats.Grid.fitted_fields, superdarn_formats.Grid.extra_fields ] self._read_darn_records(file_struct_list) self.records = dmap2dict(self._dmap_records) return self.records
def test_DmapWrite_DmapRead_dmap2dict(self): """ Test DmapWrite writing a dmap data set to be read in by DmapRead and convert to a dictionary by dmap2dict and compared. """ dmap_dict = [{'RST.version': '4.1', 'stid': 3, 'FAC.vel': np.array([2.5, 3.5, 4.0], dtype=np.float32)}, {'RST.version': '4.1', 'stid': 3, 'FAC.vel': np.array([1, 0, 1], dtype=np.int8)}, {'RST.version': '4.1', 'stid': 3, 'FAC.vel': np.array([5.7, 2.34, -0.2], dtype=np.float32)}] dmap_data = copy.deepcopy(dmap_data_sets.dmap_data) dmap_write = pydarn.DmapWrite(dmap_data) dmap_stream = dmap_write.write_dmap_stream() dmap_read = pydarn.DmapRead(dmap_stream, True) dmap_records = dmap_read.read_records() dmap_records = dmap_read.get_dmap_records self.dmap_compare(dmap_records, dmap_data) dmap_dict2 = pydarn.dmap2dict(dmap_records) self.dict_compare(dmap_dict, dmap_dict2)
def read_fitacf(self) -> List[dict]: """ Reads Fitacf DMAP file/stream Returns ------- dmap_records : List[dict] DMAP record of the Fitacf data """ pydarn_log.debug("Reading Fitacf file: {}".format(self.dmap_file)) # We need to separate the fields into subsets because fitacf fitting # methods 2.5 and 3.0 do not include a subset of fields if the data # quality is not "good". See missing_field_check method in # SDarnUtilities for more information. file_struct_list = [ superdarn_formats.Fitacf.types, superdarn_formats.Fitacf.extra_fields, superdarn_formats.Fitacf.fitted_fields, superdarn_formats.Fitacf.elevation_fields ] self._read_darn_records(file_struct_list) self.records = dmap2dict(self._dmap_records) return self.records
def plot_range_time(cls, dmap_data: List[dict], parameter: str = 'v', beam_num: int = 0, channel: int = 'all', ax=None, background: str = 'w', groundscatter: bool = False, zmin: int = None, zmax: int = None, start_time: datetime = None, end_time: datetime = None, colorbar: plt.colorbar = None, ymax: int = None, colorbar_label: str = '', norm=colors.Normalize, cmap: str = PyDARNColormaps.PYDARN_VELOCITY, filter_settings: dict = {}, date_fmt: str = '%y/%m/%d\n %H:%M', **kwargs): """ Plots a range-time parameter plot of the given field name in the dmap_data Future Work ----------- Support for other data input, like "time" dictionary key containing the datetime. However, further discussion is needed if this will be the keys name or maybe another input. Parameters ----------- dmap_data: List[dict] parameter: str key name indicating which parameter to plot. Default: v (Velocity) beam_num : int The beam number of data to plot Default: 0 channel : int or str The channel 0, 1, 2, 'all' Default : 'all' ax: matplotlib.axes axes object for another way of plotting Default: None groundscatter : boolean or str Flag to indicate if groundscatter should be plotted. If string groundscatter will be represented by that color else grey. Default : False background : str color of the background in the plot default: white zmin: int Minimum normalized value Default: minimum parameter value in the data set zmax: int Maximum normalized value Default: maximum parameter value in the data set ymax: int Sets the maximum y value Default: None, uses 'nrang' from data norm: matplotlib.colors.Normalization object This object use dependency injection to use any normalization method with the zmin and zmax. Default: colors.Normalization() start_time: datetime Start time of the plot x-axis as a datetime object Default: rounded to nearest hour-30 minutes from the first record containing the chosen parameters data end_time: datetime End time of the plot x-axis as a datetime object Default: last record of the chosen parameters data date_fmt : str format of x-axis date ticks, follow datetime format Default: '%y/%m/%d\n %H:%M' (Year/Month/Day Hour:Minute) colorbar: matplotlib.pyplot.colorbar Setting a predefined colorbar for the range-time plot If None, then a colorbar will be created for the plot Default: None colorbar_label: str the label that appears next to the color bar Default: '' cmap: str or matplotlib.cm matplotlib colour map https://matplotlib.org/tutorials/colors/colormaps.html Default: PyDARNColormaps.PYDARN_VELOCITY note: to reverse the color just add _r to the string name plot_filter: dict dictionary of the following keys for filtering data out: max_array_filter : dict dictionary that contains the key parameter names and the values to compare against. Will filter out any data points that is above this value. min_array_filter : dict dictionary that contains the key parameter names and the value to compare against. Will filter out any data points that is below this value. max_scalar_filter : dict dictionary that contains the key parameter names and the values to compare against. Will filter out data sections that is above this value. min_scalar_filter : dict dictionary that contains the key parameter names and the value to compare against. Will filter out data sections that is below this value. equal_scalar_filter : dict dictionary that contains the key parameter names and the value to compare against. Will filter out data sections that is does not equal the value. kwargs: key names of variable settings of pcolormesh: https://matplotlib.org/3.1.1/api/_as_gen/matplotlib.pyplot.pcolormesh.html Raises ------ RTPUnknownParameterError RTPIncorrectPlotMethodError RTPNoDataFoundError IndexError Returns ------- im: matplotlib.pyplot.pcolormesh matplotlib object from pcolormesh cb: matplotlib.colorbar matplotlib color bar cmap: matplotlib.cm matplotlib color map object time_axis: list list representing the x-axis datetime objects y_axis: list list representing the y-axis range gates z_data: 2D numpy array 2D array of the parameters values at the given time and range gate See Also --------- colors: https://matplotlib.org/2.0.2/api/colors_api.html color maps: PyDARNColormaps or https://matplotlib.org/tutorials/colors/colormaps.html normalize: https://matplotlib.org/3.1.1/api/_as_gen/matplotlib.colors.Normalize.html colorbar: https://matplotlib.org/3.1.1/api/_as_gen/matplotlib.pyplot.colorbar.html pcolormesh: https://matplotlib.org/3.1.1/api/_as_gen/matplotlib.pyplot.pcolormesh.html """ # Settings plot_filter = { 'min_array_filter': dict(), 'max_array_filter': dict(), 'min_scalar_filter': dict(), 'max_scalar_filter': dict(), 'equal_scalar_filter': dict() } plot_filter.update(filter_settings) # If an axes object is not passed in then store # the equivalent object in matplotlib. This allows # for variant matplotlib plotting styles. if not ax: ax = plt.gca() # Determine if a DmapRecord was passed in, instead of a list try: # because of partial records we need to find the first # record that has that parameter index_first_match = next(i for i, d in enumerate(dmap_data) if parameter in d) if isinstance(dmap_data[index_first_match][parameter], DmapArray) or\ isinstance(dmap_data[index_first_match][parameter], DmapScalar): dmap_data = dmap2dict(dmap_data) except StopIteration: raise rtp_exceptions.RTPUnknownParameterError(parameter) cls.dmap_data = dmap_data cls.__check_data_type(parameter, 'array', index_first_match) start_time, end_time = cls.__determine_start_end_time( start_time, end_time) # y-axis coordinates, i.e., range gates, # TODO: implement variant other coordinate systems for the y-axis # y shape needs to be +1 longer as requirement of how pcolormesh # draws the pixels on the grid # because nrang can change based on mode we need to look # for the largest value y_max = max(record['nrang'] for record in cls.dmap_data) y = np.arange(0, y_max + 1, 1) # z: parameter data mapped into the color mesh z = np.zeros((1, y_max)) * np.nan # x: time date data x = [] # We cannot simply use numpy's built in min and max function # because of the groundscatter value :( # These flags indicate if zmin and zmax should change set_zmin = True set_zmax = True if zmin is None: zmin = cls.dmap_data[index_first_match][parameter][0] set_zmin = False if zmax is None: zmax = cls.dmap_data[index_first_match][parameter][0] set_zmax = False for dmap_record in cls.dmap_data: # get time difference to test if there is some gap data rec_time = cls.__time2datetime(dmap_record) diff_time = 0.0 if rec_time > end_time: break if x != []: # 60.0 seconds in a minute delta_diff_time = (rec_time - x[-1]) diff_time = delta_diff_time.seconds / 60.0 # separation roughly 2 minutes if diff_time > 2.0: # if there is gap data (no data recorded past 2 minutes) # then fill it in with white space for _ in range(0, int(np.floor(diff_time / 2.0))): x.append(x[-1] + timedelta(0, 120)) i = len(x) - 1 # offset since we start at 0 not 1 if i > 0: z = np.insert(z, len(z), np.zeros(1, y_max) * np.nan, axis=0) # Get data for the provided beam number if (beam_num == 'all' or dmap_record['bmnum'] == beam_num) and\ (channel == 'all' or dmap_record['channel'] == channel): if start_time <= rec_time: # construct the x-axis array # Numpy datetime is used because it properly formats on the # x-axis x.append(rec_time) # I do this to avoid having an extra loop to just count how # many records contain the beam number i = len(x) - 1 # offset since we start at 0 not 1 # insert a new column into the z_data if i > 0: z = np.insert(z, len(z), np.zeros(1, y_max) * np.nan, axis=0) try: if len(dmap_record[parameter]) == dmap_record['nrang']: good_gates = range(len(dmap_record[parameter])) else: good_gates = dmap_record['slist'] # get the range gates that have "good" data in it for j in range(len(good_gates)): # if it is groundscatter store a very # low number in that cell if groundscatter and\ dmap_record['gflg'][j] == 1: # chosen value from davitpy to make the # groundscatter a different color # from the color map z[i][good_gates[j]] = -1000000 # otherwise store parameter value # TODO: refactor and clean up this code elif cls.__filter_data_check( dmap_record, plot_filter, j): z[i][good_gates[j]] = \ dmap_record[parameter][j] # calculate min and max value if not set_zmin and\ z[i][good_gates[j]] < zmin: zmin = z[i][good_gates[j]] if not set_zmax and \ z[i][good_gates[j]] > zmax: zmax = z[i][good_gates[j]] # a KeyError may be thrown because slist is not created # due to bad quality data. except KeyError: continue x.append(end_time) # Check if there is any data to plot if np.all(np.isnan(z)): raise rtp_exceptions.RTPNoDataFoundError(parameter, beam_num, start_time, end_time, cls.dmap_data[0]['bmnum']) time_axis, y_axis = np.meshgrid(x, y) z_data = np.ma.masked_where(np.isnan(z.T), z.T) norm = norm(zmin, zmax) if isinstance(cmap, str): cmap = cm.get_cmap(cmap) if isinstance(groundscatter, str): cmap.set_under(groundscatter, 1.0) elif groundscatter: cmap.set_under('grey', 1.0) # set the background color, this needs to happen to avoid # the overlapping problem that occurs # cmap.set_bad(color=background, alpha=1.) # plot! im = ax.pcolormesh(time_axis, y_axis, z_data, lw=0.01, cmap=cmap, norm=norm, **kwargs) # setup some standard axis information # Upon request of Daniel Billet and others, I am rounding # the time down so the plotting x-axis will show the origin # time label # TODO: may need to be its own function rounded_down_start_time = x[0] -\ timedelta(minutes=x[0].minute % 15, seconds=x[0].second, microseconds=x[0].microsecond) ax.set_xlim([rounded_down_start_time, x[-1]]) ax.xaxis.set_major_formatter(dates.DateFormatter(date_fmt)) if ymax is None: ymax = y_max ax.set_ylim(0, ymax) ax.yaxis.set_ticks(np.arange(0, ymax + 1, (ymax) / 5)) # SuperDARN file typically are in 2hr or 24 hr files # to make the minute ticks sensible, the time length is detected # then a interval is picked. 30 minute ticks for 24 hr plots # and 5 minute ticks for 2 hour plots. data_time_length = end_time - start_time # 3 hours * 60 minutes * 60 seconds if data_time_length.total_seconds() > 3 * 60 * 60: tick_interval = 30 else: tick_interval = 1 ax.xaxis.set_minor_locator(dates.MinuteLocator(interval=tick_interval)) ax.yaxis.set_minor_locator(ticker.MultipleLocator(5)) # so the plots gets to the ends ax.margins(0) # create color bar if True if not colorbar: with warnings.catch_warnings(): warnings.filterwarnings('error') try: locator = ticker.MaxNLocator(symmetric=True, min_n_ticks=3, integer=True, nbins='auto') ticks = locator.tick_values(vmin=zmin, vmax=zmax) cb = ax.figure.colorbar(im, ax=ax, extend='both', ticks=ticks) except (ZeroDivisionError, Warning): raise rtp_exceptions.RTPZeroError(parameter, beam_num, zmin, zmax, norm) from None if colorbar_label != '': cb.set_label(colorbar_label) return im, cb, cmap, x, y, z_data
def plot_time_series(cls, dmap_data: List[dict], parameter: str = 'tfreq', beam_num: int = 0, ax=None, start_time: datetime = None, end_time: datetime = None, date_fmt: str = '%y/%m/%d\n %H:%M', channel='all', scale: str = 'linear', cp_name: bool = True, **kwargs): """ Plots the time series of a scalar parameter Parameters ---------- dmap_data : List[dict] List of dictionaries representing SuperDARN data parameter : str Scalar parameter to plot Default: tfreq beam_num : int The beam number of data to plot Default: 0 ax : matplotlib axes object option to pass in axes object from matplotlib.pyplot Default: plt.gca() start_time: datetime Start time of the plot x-axis as a datetime object Default: first record date end_time: datetime End time of the plot x-axis as a datetime object Default: last record date date_fmt : datetime format string Date format for the x-axis Default: '%y/%m/%d \n %H:%M' channel : int or str integer indicating which channel to plot or 'all' to plot all channels Default: 'all' scale: str The y-axis scaling. This is not used for plotting the cp ID Default: log cp_name : bool If True, the cp ID name will be printed along side the number. Otherwise the cp ID will just be printed. This is only used for the parameter cp Default: True kwargs kwargs passed into plot_date Raises ------ RTPUnknownParameterError RTPIncorrectPlotMethodError RTPNoDataFoundError IndexError Returns ------- lines: list list of matplotlib.lines.Lines2D object representing the time-series data if plotting parameter cp then it will be None x: list list of datetime objects representing x-axis time series y: list list of scalar values for each datetime object See Also -------- yscale: https://matplotlib.org/3.1.1/api/_as_gen/matplotlib.pyplot.yscale.html plot_date: https://matplotlib.org/3.1.1/api/_as_gen/matplotlib.axes.Axes.plot_date.html colors: https://matplotlib.org/2.0.2/api/colors_api.html """ # check if axes object is passed in, if not # Default to plt.gca() if not ax: ax = plt.gca() # Determine if a DmapRecord was passed in, instead of a list try: # because of partial records we need to find the first # record that has that parameter index_first_match = next(i for i, d in enumerate(dmap_data) if parameter in d) if isinstance(dmap_data[index_first_match][parameter], DmapArray) or\ isinstance(dmap_data[index_first_match][parameter], DmapScalar): dmap_data = dmap2dict(dmap_data) except StopIteration: raise rtp_exceptions.RTPUnknownParameterError(parameter) cls.dmap_data = dmap_data cls.__check_data_type(parameter, 'scalar', index_first_match) start_time, end_time = cls.__determine_start_end_time( start_time, end_time) # initialized here for return purposes lines = None # parameter data y = [] # date time x = [] # plot CPID if parameter == 'cp': old_cpid = None for dmap_record in cls.dmap_data: # TODO: this check could be a function call x.append(cls.__time2datetime(dmap_record)) if (dmap_record['bmnum'] == beam_num or beam_num == 'all') and\ (dmap_record['channel'] == channel or channel == 'all'): rec_time = cls.__time2datetime(dmap_record) if start_time <= rec_time and rec_time <= end_time: if old_cpid != dmap_record['cp'] or old_cpid is None: ax.axvline(x=rec_time, color='black') old_cpid = dmap_record['cp'] ax.text(x=rec_time + timedelta(seconds=600), y=0.7, s=dmap_record['cp']) if cp_name: # Keeping this commented code in to show how # we could get the name from the file; however, # there is not set format for combf field ... # so we will use the dictionary to prevent # errors or incorrect names on the plot. # However, we should get it from the file # not a dictionary that might not be updated # cpid_command = # dmap_record['combf'].split(' ') # if len(cpid_command) == 1: # cp_name = cpid_command[0] # elif len(cpid_command) == 0: # cp_name = 'unknown' # else: # cp_name = cpid_command[1] if dmap_record['cp'] < 0: cpID_name = 'discretionary \n{}'\ ''.format(SuperDARNCpids.cpids. get(abs(dmap_record['cp']), 'unknown')) else: cpID_name =\ SuperDARNCpids.cpids.\ get(abs(dmap_record['cp']), 'unknown') ax.text(x=rec_time + timedelta(seconds=600), y=0.1, s=cpID_name) # Check if the old cp ID change, if not then there was no data if old_cpid is None: raise rtp_exceptions.\ RTPNoDataFoundError(parameter, beam_num, start_time, end_time, cls.dmap_data[0]['bmnum']) # to get rid of y-axis numbers ax.set_yticks([]) else: for dmap_record in cls.dmap_data: # TODO: this check could be a function call rec_time = cls.__time2datetime(dmap_record) if start_time <= rec_time and rec_time <= end_time: if (dmap_record['bmnum'] == beam_num or beam_num == 'all') and \ (channel == dmap_record['channel'] or channel == 'all'): # construct the x-axis array x.append(rec_time) if parameter == 'tfreq': # Convert kHz to MHz by dividing by 1000 y.append(dmap_record[parameter] / 1000) else: y.append(dmap_record[parameter]) # else plot missing data elif len(x) > 0: diff_time = rec_time - x[-1] # if the time difference is greater than 2 minutes # meaning no data was collected for that time period # then plot nothing. if diff_time.total_seconds() > 2.0 * 60.0: x.append(rec_time) y.append(np.nan) # for masking the data # Check if there is any data to plot if np.all(np.isnan(y)) or len(x) == 0: raise rtp_exceptions.\ RTPNoDataFoundError(parameter, beam_num, start_time, end_time, cls.dmap_data[0]['bmnum']) # using masked arrays to create gaps in the plot # otherwise the lines will connect in gapped data my = np.ma.array(y) my = np.ma.masked_where(np.isnan(my), my) lines = ax.plot_date(x, my, fmt='k', tz=None, xdate=True, ydate=False, **kwargs) rounded_down_start_time = x[0] -\ timedelta(minutes=x[0].minute % 15, seconds=x[0].second, microseconds=x[0].microsecond) ax.set_xlim([rounded_down_start_time, x[-1]]) ax.set_yscale(scale) # set date format and minor hourly locators # Rounded the time down to show origin label upon # Daniel Billet and others request. # TODO: may move this to its own function rounded_down_start_time = x[0] -\ timedelta(minutes=x[0].minute % 15, seconds=x[0].second, microseconds=x[0].microsecond) ax.set_xlim([rounded_down_start_time, x[-1]]) ax.xaxis.set_major_formatter(dates.DateFormatter(date_fmt)) ax.xaxis.set_minor_locator(dates.HourLocator()) # SuperDARN file typically are in 2hr or 24 hr files # to make the minute ticks sensible, the time length is detected # then a interval is picked. 30 minute ticks for 24 hr plots # and 5 minute ticks for 2 hour plots. data_time_length = end_time - start_time # 3 hours * 60 minutes * 60 seconds if data_time_length.total_seconds() > 3 * 60 * 60: tick_interval = 30 else: tick_interval = 1 ax.xaxis.set_minor_locator(dates.MinuteLocator(interval=tick_interval)) ax.margins(x=0) ax.tick_params(axis='y', which='minor') return lines, x, y