yVars = [s for s in varList if not reg_ex.search(s)] for v in yVars: print v t = f['time'].data t_dict = dict(data = t, info = dict(label='Time', units='GMT')) y = f[v] y_dict = dict(data = y.data, info = dict(label=v, units = y.units, var = v, platform = platform, node = node, sensor = sensor)) # plot timeseries with outliers fig, ax = pf.auto_plot(t_dict, y_dict, title, stdev=None, line_style='.', g_range=True, color=None) pf.resize(width=12, height=8.5) # Resize figure # format date axis df = mdates.DateFormatter('%Y-%m-%d') ax.xaxis.set_major_formatter(df) fig.autofmt_xdate() save_file = os.path.join(save_dir, fName + '_' + v) plt.savefig(str(save_file),dpi=150) # save figure plt.close('all') # plot timeseries with outliers removed fig, ax = pf.auto_plot(t_dict, y_dict, title, stdev=3, line_style='.', g_range=True, color=None) pf.resize(width=12, height=8.5) # Resize figure
y = f[v] y_dict = dict(data=y.data, info=dict(label=v, units=y.units, var=v, platform=platform, node=node, sensor=sensor)) # plot timeseries with outliers fig, ax = pf.auto_plot(t_dict, y_dict, title, stdev=None, line_style='.', g_range=True, color=None) pf.resize(width=12, height=8.5) # Resize figure # format date axis df = mdates.DateFormatter('%Y-%m-%d') ax.xaxis.set_major_formatter(df) fig.autofmt_xdate() save_file = os.path.join(save_dir, fName + '_' + v) plt.savefig(str(save_file), dpi=150) # save figure plt.close('all') # plot timeseries with outliers removed
def main(folder, out, time_break): """ files: url to an .nc/.ncml file or the path to a text file containing .nc/.ncml links. A # at the front will skip links in the text file. out: Directory to save plots """ with xr.open_mfdataset(folder, mask_and_scale=False) as ds: # change dimensions from 'obs' to 'time' ds = ds.swap_dims({'obs': 'time'}) ds_variables = ds.data_vars.keys() # List of dataset variables stream = ds.stream # List stream name associated with the data title_pre = mk_str(ds.attrs, 't') # , var, tt0, tt1, 't') save_pre = mk_str(ds.attrs, 's') # , var, tt0, tt1, 's') platform = ds.subsite node = ds.node sensor = ds.sensor save_dir = os.path.join(out, ds.subsite, ds.node, ds.stream, 'timeseries') cf.create_dir(save_dir) try: eng = stream_vars[ stream] # select specific streams engineering variables except KeyError: eng = [''] misc = [ 'timestamp', 'provenance', 'qc', 'id', 'obs', 'deployment', 'volts', 'counts', 'quality_flag' ] reg_ex = re.compile('|'.join(eng + misc)) # make regular expression # keep variables that are not in the regular expression sci_vars = [s for s in ds_variables if not reg_ex.search(s)] # t0, t1 = pf.get_rounded_start_and_end_times(ds_disk['time'].data) # tI = (pd.to_datetime(t0) + (pd.to_datetime(t1) - pd.to_datetime(t0)) / 2) # time_list = [[t0, t1], [t0, tI], [tI, t1]] times = np.unique(ds[time_break]) for t in times: time_ind = t == ds[time_break].data for var in sci_vars: x = dict(data=ds['time'].data[time_ind], info=dict(label='Time', units='GMT')) t0 = pd.to_datetime( x['data'].min()).strftime('%Y-%m-%dT%H%M%00') t1 = pd.to_datetime( x['data'].max()).strftime('%Y-%m-%dT%H%M%00') try: sci = ds[var] print var # sci = sub_ds[var] except UnicodeEncodeError: # some comments have latex characters ds[var].attrs.pop('comment') # remove from the attributes sci = ds[var] # or else the variable won't load # define possible pressure variables pressure_vars = [ 'seawater_pressure', 'sci_water_pressure_dbar', 'ctdgv_m_glider_instrument_recovered-sci_water_pressure_dbar', 'ctdgv_m_glider_instrument-sci_water_pressure_dbar' ] rePressure = re.compile('|'.join(pressure_vars)) # define y as pressure variable pressure = [s for s in sci.variables if rePressure.search(s)] pressure = ''.join(pressure) y = sci.variables[pressure] yN = pressure y_units = sci.units try: y_lab = sci.long_name except AttributeError: y_lab = sci.standard_name y = dict(data=sci.data[time_ind], info=dict(label=y_lab, units=sci.units, var=var, platform=platform, node=node, sensor=sensor)) title = title_pre + var # plot timeseries with outliers fig, ax = pf.auto_plot(x, y, title, stdev=None, line_style='r-o', g_range=True) pf.resize(width=12, height=8.5) # Resize figure save_name = '{}-{}-{}_{}_{}-{}'.format(platform, node, sensor, var, t0, t1) pf.save_fig(save_dir, save_name, res=150) # Save figure plt.close('all') # plot z variable each time fig, ax = pf.depth_cross_section(x, y, title, stdev=1, line_style='r-o', g_range=True) pf.resize(width=12, height=8.5) # Resize figure save_name = '{}-{}-{}_{}_{}-{}_outliers_removed'.format( platform, node, sensor, var, t0, t1) pf.save_fig(save_dir, save_name, res=150) # Save figure plt.close('all') del x, y
def main(nc, directory, out, time_break, breakdown): """ files: url to an .nc/.ncml file or the path to a text file containing .nc/.ncml links. A # at the front will skip links in the text file. out: Directory to save plots """ list_files = directory + "/*.nc" # list_files = ['https://opendap.oceanobservatories.org/thredds/dodsC/ooi/friedrich-knuth-gmail/20170322T191659-RS03AXPS-PC03A-4A-CTDPFA303-streamed-ctdpf_optode_sample/deployment0003_RS03AXPS-PC03A-4A-CTDPFA303-streamed-ctdpf_optode_sample_20170312T000000.426102-20170322T190000.059973.nc', # 'https://opendap.oceanobservatories.org/thredds/dodsC/ooi/friedrich-knuth-gmail/20170322T191659-RS03AXPS-PC03A-4A-CTDPFA303-streamed-ctdpf_optode_sample/deployment0003_RS03AXPS-PC03A-4A-CTDPFA303-streamed-ctdpf_optode_sample_20161222T000000.132709-20170311T235959.426096.nc'] # print list_files stream_vars = pf.load_variable_dict(var='eng') # load engineering variables with xr.open_dataset(nc, mask_and_scale=False) as ds_ncfile: stream = ds_ncfile.stream # List stream name associated with the data title_pre = mk_str(ds_ncfile.attrs, 't') # , var, tt0, tt1, 't') save_pre = mk_str(ds_ncfile.attrs, 's') # , var, tt0, tt1, 's') platform = ds_ncfile.subsite node = ds_ncfile.node sensor = ds_ncfile.sensor # save_dir = os.path.join(out, platform, node, stream, 'xsection_depth_profiles') save_dir = os.path.join(out,'timeseries',breakdown) cf.create_dir(save_dir) with xr.open_mfdataset(list_files) as ds: # change dimensions from 'obs' to 'time' ds = ds.swap_dims({'obs': 'time'}) ds_variables = ds.data_vars.keys() # List of dataset variables # try: # eng = stream_vars[stream] # select specific streams engineering variables # except KeyError: # eng = [''] misc = ['quality', 'string', 'timestamp', 'deployment', 'id', 'provenance', 'qc', 'time', 'mission', 'obs', 'volt', 'ref', 'sig', 'amp', 'rph', 'calphase', 'phase', 'therm'] # reg_ex = re.compile('|'.join(eng+misc)) # make regular expression reg_ex = re.compile('|'.join(misc)) # keep variables that are not in the regular expression sci_vars = [s for s in ds_variables if not reg_ex.search(s)] # t0, t1 = pf.get_rounded_start_and_end_times(ds_disk['time'].data) # tI = (pd.to_datetime(t0) + (pd.to_datetime(t1) - pd.to_datetime(t0)) / 2) # time_list = [[t0, t1], [t0, tI], [tI, t1]] times = np.unique(ds[time_break]) for t in times: time_ind = t == ds[time_break].data for var in sci_vars: x = dict(data=ds['time'].data[time_ind], info=dict(label='Time', units='GMT')) t0 = pd.to_datetime(x['data'].min()).strftime('%Y-%m-%dT%H%M%00') t1 = pd.to_datetime(x['data'].max()).strftime('%Y-%m-%dT%H%M%00') try: sci = ds[var] print var # sci = sub_ds[var] except UnicodeEncodeError: # some comments have latex characters ds[var].attrs.pop('comment') # remove from the attributes sci = ds[var] # or else the variable won't load try: y_lab = sci.long_name except AttributeError: y_lab = sci.standard_name y = dict(data=sci.data[time_ind], info=dict(label=y_lab, units=str(sci.units), var=var, platform=platform, node=node, sensor=sensor)) title = title_pre + var # plot timeseries with outliers fig, ax = pf.auto_plot(x, y, title, stdev=None, line_style='.', g_range=True) pf.resize(width=12, height=8.5) # Resize figure save_name = '{}-{}-{}_{}_{}-{}'.format(platform, node, sensor, var, t0, t1) pf.save_fig(save_dir, save_name, res=150) # Save figure plt.close('all') # try: # y_lab = sci.standard_name # except AttributeError: # y_lab = var # y = dict(data=sci.data, info=dict(label=y_lab, units=sci.units)) # plot timeseries with outliers removed # fig, ax = pf.auto_plot(x, y, title, stdev=1, line_style='.', g_range=True) # pf.resize(width=12, height=8.5) # Resize figure # save_name = '{}-{}-{}_{}_{}-{}_outliers_removed'.format(platform, node, sensor, var, t0, t1) # pf.save_fig(save_dir, save_name, res=150) # Save figure # plt.close('all') del x, y
def main(nc, directory, out, time_break, breakdown): """ files: url to an .nc/.ncml file or the path to a text file containing .nc/.ncml links. A # at the front will skip links in the text file. out: Directory to save plots """ list_files = directory + "/*.nc" # list_files = ['https://opendap.oceanobservatories.org/thredds/dodsC/ooi/friedrich-knuth-gmail/20170322T191659-RS03AXPS-PC03A-4A-CTDPFA303-streamed-ctdpf_optode_sample/deployment0003_RS03AXPS-PC03A-4A-CTDPFA303-streamed-ctdpf_optode_sample_20170312T000000.426102-20170322T190000.059973.nc', # 'https://opendap.oceanobservatories.org/thredds/dodsC/ooi/friedrich-knuth-gmail/20170322T191659-RS03AXPS-PC03A-4A-CTDPFA303-streamed-ctdpf_optode_sample/deployment0003_RS03AXPS-PC03A-4A-CTDPFA303-streamed-ctdpf_optode_sample_20161222T000000.132709-20170311T235959.426096.nc'] # print list_files stream_vars = pf.load_variable_dict( var='eng') # load engineering variables with xr.open_dataset(nc, mask_and_scale=False) as ds_ncfile: stream = ds_ncfile.stream # List stream name associated with the data title_pre = mk_str(ds_ncfile.attrs, 't') # , var, tt0, tt1, 't') save_pre = mk_str(ds_ncfile.attrs, 's') # , var, tt0, tt1, 's') platform = ds_ncfile.subsite node = ds_ncfile.node sensor = ds_ncfile.sensor # save_dir = os.path.join(out, platform, node, stream, 'xsection_depth_profiles') save_dir = os.path.join(out, 'timeseries', breakdown) cf.create_dir(save_dir) with xr.open_mfdataset(list_files) as ds: # change dimensions from 'obs' to 'time' ds = ds.swap_dims({'obs': 'time'}) ds_variables = ds.data_vars.keys() # List of dataset variables # try: # eng = stream_vars[stream] # select specific streams engineering variables # except KeyError: # eng = [''] misc = [ 'quality', 'string', 'timestamp', 'deployment', 'id', 'provenance', 'qc', 'time', 'mission', 'obs', 'volt', 'ref', 'sig', 'amp', 'rph', 'calphase', 'phase', 'therm' ] # reg_ex = re.compile('|'.join(eng+misc)) # make regular expression reg_ex = re.compile('|'.join(misc)) # keep variables that are not in the regular expression sci_vars = [s for s in ds_variables if not reg_ex.search(s)] # t0, t1 = pf.get_rounded_start_and_end_times(ds_disk['time'].data) # tI = (pd.to_datetime(t0) + (pd.to_datetime(t1) - pd.to_datetime(t0)) / 2) # time_list = [[t0, t1], [t0, tI], [tI, t1]] times = np.unique(ds[time_break]) for t in times: time_ind = t == ds[time_break].data for var in sci_vars: x = dict(data=ds['time'].data[time_ind], info=dict(label='Time', units='GMT')) t0 = pd.to_datetime( x['data'].min()).strftime('%Y-%m-%dT%H%M%00') t1 = pd.to_datetime( x['data'].max()).strftime('%Y-%m-%dT%H%M%00') try: sci = ds[var] print var # sci = sub_ds[var] except UnicodeEncodeError: # some comments have latex characters ds[var].attrs.pop('comment') # remove from the attributes sci = ds[var] # or else the variable won't load try: y_lab = sci.long_name except AttributeError: y_lab = sci.standard_name y = dict(data=sci.data[time_ind], info=dict(label=y_lab, units=str(sci.units), var=var, platform=platform, node=node, sensor=sensor)) title = title_pre + var # plot timeseries with outliers fig, ax = pf.auto_plot(x, y, title, stdev=None, line_style='.', g_range=True) pf.resize(width=12, height=8.5) # Resize figure save_name = '{}-{}-{}_{}_{}-{}'.format(platform, node, sensor, var, t0, t1) pf.save_fig(save_dir, save_name, res=150) # Save figure plt.close('all') # try: # y_lab = sci.standard_name # except AttributeError: # y_lab = var # y = dict(data=sci.data, info=dict(label=y_lab, units=sci.units)) # plot timeseries with outliers removed # fig, ax = pf.auto_plot(x, y, title, stdev=1, line_style='.', g_range=True) # pf.resize(width=12, height=8.5) # Resize figure # save_name = '{}-{}-{}_{}_{}-{}_outliers_removed'.format(platform, node, sensor, var, t0, t1) # pf.save_fig(save_dir, save_name, res=150) # Save figure # plt.close('all') del x, y
def main(folder, out, time_break): """ files: url to an .nc/.ncml file or the path to a text file containing .nc/.ncml links. A # at the front will skip links in the text file. out: Directory to save plots """ with xr.open_mfdataset(folder, mask_and_scale=False) as ds: # change dimensions from 'obs' to 'time' ds = ds.swap_dims({'obs': 'time'}) ds_variables = ds.data_vars.keys() # List of dataset variables stream = ds.stream # List stream name associated with the data title_pre = mk_str(ds.attrs, 't') # , var, tt0, tt1, 't') save_pre = mk_str(ds.attrs, 's') # , var, tt0, tt1, 's') platform = ds.subsite node = ds.node sensor = ds.sensor save_dir = os.path.join(out, ds.subsite, ds.node, ds.stream, 'timeseries') cf.create_dir(save_dir) try: eng = stream_vars[stream] # select specific streams engineering variables except KeyError: eng = [''] misc = ['timestamp', 'provenance', 'qc', 'id', 'obs', 'deployment', 'volts', 'counts', 'quality_flag'] reg_ex = re.compile('|'.join(eng+misc)) # make regular expression # keep variables that are not in the regular expression sci_vars = [s for s in ds_variables if not reg_ex.search(s)] # t0, t1 = pf.get_rounded_start_and_end_times(ds_disk['time'].data) # tI = (pd.to_datetime(t0) + (pd.to_datetime(t1) - pd.to_datetime(t0)) / 2) # time_list = [[t0, t1], [t0, tI], [tI, t1]] times = np.unique(ds[time_break]) for t in times: time_ind = t == ds[time_break].data for var in sci_vars: x = dict(data=ds['time'].data[time_ind], info=dict(label='Time', units='GMT')) t0 = pd.to_datetime(x['data'].min()).strftime('%Y-%m-%dT%H%M%00') t1 = pd.to_datetime(x['data'].max()).strftime('%Y-%m-%dT%H%M%00') try: sci = ds[var] print var # sci = sub_ds[var] except UnicodeEncodeError: # some comments have latex characters ds[var].attrs.pop('comment') # remove from the attributes sci = ds[var] # or else the variable won't load # define possible pressure variables pressure_vars = ['seawater_pressure', 'sci_water_pressure_dbar', 'ctdgv_m_glider_instrument_recovered-sci_water_pressure_dbar', 'ctdgv_m_glider_instrument-sci_water_pressure_dbar'] rePressure = re.compile('|'.join(pressure_vars)) # define y as pressure variable pressure = [s for s in sci.variables if rePressure.search(s)] pressure = ''.join(pressure) y = sci.variables[pressure] yN = pressure y_units = sci.units try: y_lab = sci.long_name except AttributeError: y_lab = sci.standard_name y = dict(data=sci.data[time_ind], info=dict(label=y_lab, units=sci.units, var=var, platform=platform, node=node, sensor=sensor)) title = title_pre + var # plot timeseries with outliers fig, ax = pf.auto_plot(x, y, title, stdev=None, line_style='r-o', g_range=True) pf.resize(width=12, height=8.5) # Resize figure save_name = '{}-{}-{}_{}_{}-{}'.format(platform, node, sensor, var, t0, t1) pf.save_fig(save_dir, save_name, res=150) # Save figure plt.close('all') # plot z variable each time fig, ax = pf.depth_cross_section(x, y, title, stdev=1, line_style='r-o', g_range=True) pf.resize(width=12, height=8.5) # Resize figure save_name = '{}-{}-{}_{}_{}-{}_outliers_removed'.format(platform, node, sensor, var, t0, t1) pf.save_fig(save_dir, save_name, res=150) # Save figure plt.close('all') del x, y
def main(files, out, start, end, time_break='full', stdev=None, color=None, interactive=False): """ files: url to an .nc/.ncml file or the path to a text file containing .nc/.ncml links. A # at the front will skip links in the text file. out: Directory to save plots """ fname, ext = os.path.splitext(files) if ext in '.nc': list_files = [files] elif ext in '.ncml': list_files = [files] else: list_files = read_file(files) stream_vars = pf.load_variable_dict( var='eng') # load engineering variables for nc in list_files: print nc with xr.open_dataset(nc, mask_and_scale=False) as ds: # change dimensions from 'obs' to 'time' ds = ds.swap_dims({'obs': 'time'}) ds_variables = ds.data_vars.keys() # List of dataset variables title_pre = mk_str(ds.attrs, 't') # , var, tt0, tt1, 't') platform = ds.subsite node = ds.node sensor = ds.sensor deployment = 'D0000{}'.format(str(np.unique(ds.deployment)[0])) stream = ds.stream save_dir = os.path.join(out, platform, deployment, node, sensor, stream, 'timeseries') cf.create_dir(save_dir) misc = [ 'quality', 'string', 'timestamp', 'deployment', 'id', 'provenance', 'qc', 'time', 'mission', 'obs', 'volt', 'ref', 'sig', 'amp', 'rph', 'calphase', 'phase', 'therm', 'description' ] # reg_ex = re.compile('|'.join(eng+misc)) # make regular expression reg_ex = re.compile('|'.join(misc)) # keep variables that are not in the regular expression sci_vars = [s for s in ds_variables if not reg_ex.search(s)] # if not time_break is 'full' and not None: # times = np.unique(ds[time_break]) # else: # times = [0] if not time_break == None: times = np.unique(ds[time_break]) for t in times: if not time_break is 'full': time_ind = t == ds[time_break].data else: time_ind = np.ones( ds['time'].data.shape, dtype=bool) # index all times to be set to True for var in sci_vars: x = dict(data=ds['time'].data[time_ind], info=dict(label='Time', units='GMT')) t0 = pd.to_datetime( x['data'].min()).strftime('%Y-%m-%dT%H%M%00') t1 = pd.to_datetime( x['data'].max()).strftime('%Y-%m-%dT%H%M%00') try: sci = ds[var] print var except UnicodeEncodeError: # some comments have latex characters ds[var].attrs.pop( 'comment') # remove from the attributes sci = ds[var] # or else the variable won't load try: y_lab = sci.long_name except AttributeError: try: y_lab = sci.standard_name except AttributeError: print 'attribute list is not complete. long_ and standard_ names are not available' y_lab = var try: y_unit = str(sci.units) except AttributeError: print 'attribute list is not complete. missing units' y_unit = 'missing' y = dict(data=sci.data[time_ind], info=dict(label=y_lab, units=y_unit, var=var, platform=platform, node=node, sensor=sensor)) title = title_pre + var # plot timeseries with outliers fig, ax = pf.auto_plot(x, y, title, stdev=None, line_style='.', g_range=True, color=color) pf.resize(width=12, height=8.5) # Resize figure save_name = '{}-{}-{}_{}_{}-{}'.format( platform, node, sensor, var, t0, t1) pf.save_fig(save_dir, save_name, res=150) # Save figure plt.close('all') # plot timeseries with outliers removed fig, ax = pf.auto_plot(x, y, title, stdev=stdev, line_style='.', g_range=True, color=color) pf.resize(width=12, height=8.5) # Resize figure save_name = '{}-{}-{}_{}_{}-{}_outliers_removed'.format( platform, node, sensor, var, t0, t1) pf.save_fig(save_dir, save_name, res=150) # Save figure plt.close('all') # del x, y else: ds = ds.sel(time=slice(start, end)) for var in sci_vars: x = dict(data=ds['time'].data[:], info=dict(label='Time', units='GMT')) t0 = pd.to_datetime( x['data'].min()).strftime('%Y-%m-%dT%H%M%00') t1 = pd.to_datetime( x['data'].max()).strftime('%Y-%m-%dT%H%M%00') try: sci = ds[var] print var except UnicodeEncodeError: # some comments have latex characters ds[var].attrs.pop( 'comment') # remove from the attributes sci = ds[var] # or else the variable won't load try: y_lab = sci.long_name except AttributeError: try: y_lab = sci.standard_name except AttributeError: print 'attribute list is not complete. long_ and standard_ names are not available' y_lab = var try: y_unit = sci.units except AttributeError: print 'attribute list is not complete. units are missing' y_unit = 'missing' y = dict(data=sci.data[:], info=dict(label=y_lab, units=y_unit, var=var, platform=platform, node=node, sensor=sensor)) title = title_pre + var # plot timeseries with outliers fig, ax = pf.auto_plot(x, y, title, stdev=None, line_style='.', g_range=True, color=color) pf.resize(width=12, height=8.5) # Resize figure save_name = '{}-{}-{}_{}_{}-{}'.format( platform, node, sensor, var, t0, t1) pf.save_fig(save_dir, save_name, res=150) # Save figure plt.close('all') # plot timeseries with outliers removed fig, ax = pf.auto_plot(x, y, title, stdev=stdev, line_style='.', g_range=True, color=color, interactive=interactive) if interactive == True: fig.canvas.mpl_connect( 'pick_event', lambda event: pf.onpick2( event, x['data'], y['data'])) plt.show() else: pf.resize(width=12, height=8.5) # Resize figure save_name = '{}-{}-{}_{}_{}-{}_outliers_removed'.format( platform, node, sensor, var, t0, t1) pf.save_fig(save_dir, save_name, res=150) # Save figure plt.close('all')