def main(argv=None): # setup the input arguments args = inputs(argv) site = args.site node = args.node sensor = args.sensor method = args.method stream = args.stream deploy = args.deploy start = args.start stop = args.stop # determine the start and stop times for the data request based on either the deployment number or user entered # beginning and ending dates. if not deploy or (start and stop): return SyntaxError('You must specify either a deployment number or beginning and end dates of interest.') else: if deploy: # Determine start and end dates based on the deployment number start, stop = get_deployment_dates(site, node, sensor, deploy) if not start or not stop: exit_text = ('Deployment dates are unavailable for %s-%s-%s, deployment %02d.' % (site, node, sensor, deploy)) raise SystemExit(exit_text) # Request the data for download r = m2m_request(site, node, sensor, method, stream, start, stop) if not r: exit_text = ('Request failed for %s-%s-%s. Check request.' % (site, node, sensor)) raise SystemExit(exit_text) # Valid request, start downloading the data if deploy: phsen = m2m_collect(r, ('.*deployment%04d.*PHSEN.*\\.nc$' % deploy)) else: phsen = m2m_collect(r, '.*PHSEN.*\\.nc$') if not phsen: exit_text = ('Data unavailable for %s-%s-%s. Check request.' % (site, node, sensor)) raise SystemExit(exit_text) # clean-up and reorganize if method in ['telemetered', 'recovered_host']: if re.match('.*imodem.*', stream): phsen = phsen_imodem(phsen) else: phsen = phsen_datalogger(phsen) else: phsen = phsen_instrument(phsen) vocab = get_vocabulary(site, node, sensor)[0] phsen = update_dataset(phsen, vocab['maxdepth']) # save the data to disk out_file = os.path.abspath(args.outfile) if not os.path.exists(os.path.dirname(out_file)): os.makedirs(os.path.dirname(out_file)) phsen.to_netcdf(out_file, mode='w', format='NETCDF4', engine='h5netcdf', encoding=ENCODINGS)
def main(): # Setup needed parameters for the request, the user would need to vary these to suit their own needs and # sites/instruments of interest. Site, node, sensor, stream and delivery method names can be obtained from the # Ocean Observatories Initiative web site. The last two parameters (level and instrmt) will set path and naming # conventions to save the data to the local disk. site = 'CE02SHSM' # OOI Net site designator node = 'SBD12' # OOI Net node designator sensor = '04-PCO2AA000' # OOI Net sensor designator stream = 'pco2a_a_dcl_instrument_air' # OOI Net stream name method = 'telemetered' # OOI Net data delivery method level = 'buoy' # local directory name, level below site instrmt = 'pco2a' # local directory name, instrument below level # We are after telemetered data. Determine list of deployments and use the last, presumably currently active, # deployment to determine the start and end dates for our request. vocab = get_vocabulary(site, node, sensor)[0] deployments = list_deployments(site, node, sensor) deploy = deployments[-1] start, stop = get_deployment_dates(site, node, sensor, deploy) # request and download the data -- air measurements r = m2m_request(site, node, sensor, method, stream, start, stop) air = m2m_collect(r, ('.*deployment%04d.*PCO2A.*air.*\\.nc$' % deploy)) # request and download the data -- water measurements r = m2m_request(site, node, sensor, method, 'pco2a_a_dcl_instrument_water', start, stop) water = m2m_collect(r, ('.*deployment%04d.*PCO2A.*water.*\\.nc$' % deploy)) # clean-up and reorganize the air and water datasets air = pco2a_datalogger(air, True) air = update_dataset(air, vocab['maxdepth']) water = pco2a_datalogger(water, True) water = update_dataset(water, vocab['maxdepth']) # save the data -- utilize groups for the air and water datasets out_path = os.path.join(CONFIG['base_dir']['m2m_base'], site.lower(), level, instrmt) out_path = os.path.abspath(out_path) if not os.path.exists(out_path): os.makedirs(out_path) out_file = ('%s.%s.%s.deploy%02d.%s.%s.nc' % (site.lower(), level, instrmt, deploy, method, stream)) nc_out = os.path.join(out_path, out_file) air.to_netcdf(nc_out, mode='w', format='NETCDF4', engine='h5netcdf', encoding=ENCODINGS, group='air') water.to_netcdf(nc_out, mode='a', format='NETCDF4', engine='h5netcdf', encoding=ENCODINGS, group='water')
def main(): # Setup needed parameters for the request, the user would need to vary these to suit their own needs and # sites/instruments of interest. Site, node, sensor, stream and delivery method names can be obtained from the # Ocean Observatories Initiative web site. The last two will set path and naming conventions to save the data # to the local disk site = 'CE02SHSM' # OOI Net site designator node = 'RID26' # OOI Net node designator sensor = '06-PHSEND000' # OOI Net sensor designator stream = 'phsen_abcdef_dcl_instrument' # OOI Net stream name method = 'telemetered' # OOI Net data delivery method level = 'nsif' # local directory name, level below site instrmt = 'phsen' # local directory name, instrument below level # We are after telemetered data. Determine list of deployments and use the last, presumably currently active, # deployment to determine the start and end dates for our request. vocab = get_vocabulary(site, node, sensor)[0] deployments = list_deployments(site, node, sensor) deploy = deployments[-1] start, stop = get_deployment_dates(site, node, sensor, deploy) # request and download the data r = m2m_request(site, node, sensor, method, stream, start, stop) phsen = m2m_collect(r, '.*PHSEN.*\\.nc$') phsen = phsen.where(phsen.deployment == deploy, drop=True) # limit to the deployment of interest # clean-up and reorganize phsen = phsen_datalogger(phsen) phsen = update_dataset(phsen, vocab['maxdepth']) # save the data out_path = os.path.join(CONFIG['base_dir']['m2m_base'], site.lower(), level, instrmt) out_path = os.path.abspath(out_path) if not os.path.exists(out_path): os.makedirs(out_path) out_file = ('%s.%s.%s.deploy%02d.%s.%s.nc' % (site.lower(), level, instrmt, deploy, method, stream)) nc_out = os.path.join(out_path, out_file) phsen.to_netcdf(nc_out, mode='w', format='NETCDF4', engine='h5netcdf', encoding=ENCODINGS)
def main(): # Setup needed parameters for the request, the user would need to vary these to suit their own needs and # sites/instruments of interest. Site, node, sensor, stream and delivery method names can be obtained from the # Ocean Observatories Initiative web site. The last two will set path and naming conventions to save the data # to the local disk site = 'CE01ISSM' # OOI Net site designator node = 'SBD17' # OOI Net node designator sensor = '06-FLORTD000' # OOI Net sensor designator stream = 'flort_sample' # OOI Net stream name method = 'recovered_inst' # OOI Net data delivery method level = 'buoy' # local directory name, level below site instrmt = 'flort' # local directory name, instrument below level # We are after recovered instrument data. Determine list of deployments and use a more recent deployment to # determine the start and end dates for our request. vocab = get_vocabulary(site, node, sensor)[0] deployments = list_deployments(site, node, sensor) deploy = deployments[5] start, stop = get_deployment_dates(site, node, sensor, deploy) # request and download the data r = m2m_request(site, node, sensor, method, stream, start, stop) flort = m2m_collect(r, '.*FLORT.*\\.nc$') flort = flort.where(flort.deployment == deploy, drop=True) # limit to the deployment of interest # clean-up and reorganize flort = flort_instrument(flort) flort = update_dataset(flort, vocab['maxdepth']) # save the data out_path = os.path.join(CONFIG['base_dir']['m2m_base'], site.lower(), level, instrmt) out_path = os.path.abspath(out_path) if not os.path.exists(out_path): os.makedirs(out_path) out_file = ('%s.%s.%s.deploy%02d.%s.%s.nc' % (site.lower(), level, instrmt, deploy, method, stream)) nc_out = os.path.join(out_path, out_file) flort.to_netcdf(nc_out, mode='w', format='NETCDF4', engine='h5netcdf', encoding=ENCODINGS)
def main(): # Setup needed parameters for the request, the user would need to vary these to suit their own needs and # sites/instruments of interest. Site, node, sensor and stream names can be obtained from the Ocean Observatories # Initiative web site site = 'CE01ISSM' # OOI Net site designator node = 'RID16' # OOI Net node designator sensor = '05-PCO2WB000' # OOI Net sensor designator stream = 'pco2w_abc_instrument' # OOI Net stream name method = 'recovered_inst' # OOI Net data delivery method level = 'nsif' # local directory name, level below site instrmt = 'pco2w' # local directory name, instrument below level # We are after recovered host data. Determine list of deployments and use a previous one to determine the # start and end dates for our request. vocab = get_vocabulary(site, node, sensor)[0] deployments = list_deployments(site, node, sensor) deploy = deployments[-3] start, stop = get_deployment_dates(site, node, sensor, deploy) # request and download the data r = m2m_request(site, node, sensor, method, stream, start, stop) pco2w = m2m_collect(r, '^(?!.*blank).*PCO2W.*nc$') pco2w = pco2w.where(pco2w.deployment == deploy, drop=True) # limit to the deployment of interest # clean-up and reorganize pco2w = pco2w_instrument(pco2w) pco2w = update_dataset(pco2w, vocab['maxdepth']) # save the data out_path = os.path.join(CONFIG['base_dir']['m2m_base'], site.lower(), level, instrmt) out_path = os.path.abspath(out_path) if not os.path.exists(out_path): os.makedirs(out_path) out_file = ('%s.%s.%s.deploy%02d.%s.%s.nc' % (site.lower(), level, instrmt, deploy, method, stream)) nc_out = os.path.join(out_path, out_file) pco2w.to_netcdf(nc_out, mode='w', format='NETCDF4', engine='h5netcdf', encoding=ENCODINGS)
def main(): # Setup needed parameters for the request, the user would need to vary these to suit their own needs and # sites/instruments of interest. Site, node, sensor, stream and delivery method names can be obtained from the # Ocean Observatories Initiative web site. The last two parameters (level and instrmt) will set path and naming # conventions to save the data to the local disk. site = 'CE02SHSP' # OOI Net site designator node = 'SP001' # OOI Net node designator sensor = '07-FLORTJ000' # OOI Net sensor designator stream = 'flort_sample' # OOI Net stream name method = 'recovered_cspp' # OOI Net data delivery method instrmt = 'flort' # local directory name, instrument below site # We are after the recovered data. Determine list of deployments and use data from one of the earlier deployments vocab = get_vocabulary(site, node, sensor)[0] deployments = list_deployments(site, node, sensor) deploy = deployments[-4] # download the data from the Gold Copy THREDDS server flort = load_gc_thredds(site, node, sensor, method, stream, ('.*deployment%04d.*FLORT.*\\.nc$' % deploy)) # clean-up and reorganize flort = flort_cspp(flort) flort = update_dataset(flort, vocab['maxdepth']) # save the data out_path = os.path.join(CONFIG['base_dir']['m2m_base'], site.lower(), instrmt) out_path = os.path.abspath(out_path) if not os.path.exists(out_path): os.makedirs(out_path) out_file = ('%s.%s.deploy%02d.%s.%s.nc' % (site.lower(), instrmt, deploy, method, stream)) nc_out = os.path.join(out_path, out_file) flort.to_netcdf(nc_out, mode='w', format='NETCDF4', engine='h5netcdf', encoding=ENCODINGS)
def generate_qartod(site, node, sensor, cut_off): """ Load all FLORT data for a defined reference designator (using the site, node and sensor names to construct the reference designator) and collected via the different data delivery methods and combine them into a single data set from which QARTOD test limits for the gross range and climatology tests can be calculated. :param site: Site designator, extracted from the first part of the reference designator :param node: Node designator, extracted from the second part of the reference designator :param sensor: Sensor designator, extracted from the third and fourth part of the reference designator :param cut_off: string formatted date to use as cut-off for data to add to QARTOD test sets :return gr_lookup: CSV formatted strings to save to a csv file for the QARTOD gross range lookup tables. :return clm_lookup: CSV formatted strings to save to a csv file for the QARTOD climatology lookup tables. :return clm_table: CSV formatted strings to save to a csv file for the QARTOD climatology range tables. """ # load the combined data for the different sources of FLORT data data = combine_delivery_methods(site, node, sensor) # create boolean arrays of the data marked as "fail" by the quality checks and generate initial # HITL annotations that can be combined with system annotations to create a cleaned up data set # prior to calculating the QARTOD test values if node == 'WFP01': index = 10 # decimate the WFP data so we can process it else: index = 1 chl_fail = data.estimated_chlorophyll_qc_summary_flag.where( data.estimated_chlorophyll_qc_summary_flag > 3).notnull() blocks = identify_blocks(chl_fail[::index], [18, 72]) chl_hitl = create_annotations(site, node, sensor, blocks) chl_hitl['parameters'] = [[22, 1141] for i in chl_hitl['parameters']] cdom_fail = data.fluorometric_cdom_qc_summary_flag.where( data.fluorometric_cdom_qc_summary_flag > 3).notnull() blocks = identify_blocks(cdom_fail[::index], [18, 72]) cdom_hitl = create_annotations(site, node, sensor, blocks) cdom_hitl['parameters'] = [[23, 1143] for i in cdom_hitl['parameters']] beta_fail = data.beta_700_qc_summary_flag.where( data.beta_700_qc_summary_flag > 3).notnull() blocks = identify_blocks(beta_fail[::index], [18, 72], 24) beta_hitl = create_annotations(site, node, sensor, blocks) beta_hitl['parameters'] = [[24, 25, 1139] for i in beta_hitl['parameters']] # combine the different dictionaries into a single HITL annotation dictionary for later use hitl = chl_hitl.copy() for d in (cdom_hitl, beta_hitl): for key, value in d.items(): hitl[key] = hitl[key] + d[key] # get the current system annotations for the sensor annotations = get_annotations(site, node, sensor) annotations = pd.DataFrame(annotations) if not annotations.empty: annotations = annotations.drop(columns=['@class']) annotations['beginDate'] = pd.to_datetime( annotations.beginDT, unit='ms').dt.strftime('%Y-%m-%dT%H:%M:%S') annotations['endDate'] = pd.to_datetime( annotations.endDT, unit='ms').dt.strftime('%Y-%m-%dT%H:%M:%S') # append the fail annotations to the existing annotations annotations = annotations.append(pd.DataFrame(hitl), ignore_index=True, sort=False) # create an annotation-based quality flag data = add_annotation_qc_flags(data, annotations) # clean-up the data, NaN-ing values that were marked as fail in the QC checks and/or identified as a block # of failed data, and then removing all records where the rollup annotation (every parameter fails) was # set to fail. data['estimated_chlorophyll'][chl_fail] = np.nan if 'fluorometric_chl_a_annotations_qc_results' in data.variables: m = data.fluorometric_chl_a_annotations_qc_results == 4 data['estimated_chlorophyll'][m] = np.nan data['fluorometric_cdom'][cdom_fail] = np.nan if 'fluorometric_cdom_annotations_qc_results' in data.variables: m = data.fluorometric_cdom_annotations_qc_results == 4 data['fluorometric_cdom'][m] = np.nan data['beta_700'][beta_fail] = np.nan if 'total_volume_scattering_coefficient_annotations_qc_results' in data.variables: m = data.total_volume_scattering_coefficient_annotations_qc_results == 4 data['beta_700'][m] = np.nan data['bback'][m] = np.nan if 'rollup_annotations_qc_results' in data.variables: data = data.where(data.rollup_annotations_qc_results < 4) # if a cut_off date was used, limit data to all data collected up to the cut_off date. # otherwise, set the limit to the range of the downloaded data. if cut_off: cut = parser.parse(cut_off) cut = cut.astimezone(pytz.utc) end_date = cut.strftime('%Y-%m-%dT%H:%M:%S') src_date = cut.strftime('%Y-%m-%d') else: cut = parser.parse(data.time_coverage_end) cut = cut.astimezone(pytz.utc) end_date = cut.strftime('%Y-%m-%dT%H:%M:%S') src_date = cut.strftime('%Y-%m-%d') data = data.sel(time=slice('2014-01-01T00:00:00', end_date)) # set the parameters and the gross range limits parameters = ['bback', 'estimated_chlorophyll', 'fluorometric_cdom'] limits = [[0, 3], [0, 30], [0, 375]] # create the initial gross range entry gr_lookup = process_gross_range(data, parameters, limits, site=site, node=node, sensor=sensor, stream='flort_sample') # add the stream name and the source comment gr_lookup['notes'] = ( 'User range based on data collected through {}.'.format(src_date)) # based on the site and node, determine if we need a depth based climatology depth_bins = np.array([]) if node in ['SP001', 'WFP01']: if site in [ 'CE01ISSP', 'CE02SHSP', 'CE06ISSP', 'CE07SHSP', 'CE09OSPM' ]: vocab = get_vocabulary(site, node, sensor)[0] max_depth = vocab['maxdepth'] depth_bins = woa_standard_bins() m = depth_bins[:, 1] <= max_depth depth_bins = depth_bins[m, :] # create and format the climatology lookups and tables for the data clm_lookup, clm_table = process_climatology(data, parameters, limits, depth_bins=depth_bins, site=site, node=node, sensor=sensor, stream='flort_sample') # add the stream name clm_lookup['stream'] = 'flort_sample' return annotations, gr_lookup, clm_lookup, clm_table
def main(argv=None): args = inputs(argv) site = args.site node = args.node sensor = args.sensor method = args.method stream = args.stream deploy = args.deploy start = args.start stop = args.stop burst = args.burst sensor_type = args.sensor_type # determine the start and stop times for the data request based on either the deployment number or user entered # beginning and ending dates. if not deploy or (start and stop): return SyntaxError( 'You must specify either a deployment number or beginning and end dates of interest.' ) else: if deploy: # Determine start and end dates based on the deployment number start, stop = get_deployment_dates(site, node, sensor, deploy) if not start or not stop: exit_text = ( 'Deployment dates are unavailable for %s-%s-%s, deployment %02d.' % (site, node, sensor, deploy)) raise SystemExit(exit_text) # Request the data for download r = m2m_request(site, node, sensor, method, stream, start, stop) if not r: exit_text = ('Request failed for %s-%s-%s. Check request.' % (site, node, sensor)) raise SystemExit(exit_text) # Valid request, start downloading the data if deploy: dosta = m2m_collect(r, ('.*deployment%04d.*DOSTA.*\\.nc$' % deploy)) else: dosta = m2m_collect(r, '.*DOSTA.*\\.nc$') if not dosta: exit_text = ('Data unavailable for %s-%s-%s. Check request.' % (site, node, sensor)) raise SystemExit(exit_text) if not sensor_type in ['solo', 'ctdbp']: exit_text = 'You need to specify the type of DOSTA in order to process: solo or ctdbp' raise SystemExit(exit_text) # clean-up and reorganize based on the type and data delivery method if sensor_type == 'solo': dosta = dosta_datalogger(dosta, burst) if sensor_type == 'ctdbp': if method in ['telemetered', 'recovered_host']: dosta = dosta_ctdbp_datalogger(dosta) else: dosta = dosta_ctdbp_instrument(dosta) vocab = get_vocabulary(site, node, sensor)[0] dosta = update_dataset(dosta, vocab['maxdepth']) # save the data to disk out_file = os.path.abspath(args.outfile) if not os.path.exists(os.path.dirname(out_file)): os.makedirs(os.path.dirname(out_file)) dosta.to_netcdf(out_file, mode='w', format='NETCDF4', engine='h5netcdf', encoding=ENCODINGS)
def main(argv=None): args = inputs(argv) site = args.site node = args.node sensor = args.sensor method = args.method stream = args.stream deploy = args.deploy start = args.start stop = args.stop burst = args.burst # check if we are specifying a deployment or a specific date and time range if not deploy or (start and stop): return SyntaxError( 'You must specify either a deployment number or beginning and end dates of interest.' ) # if we are specifying a deployment number, then get the data from the Gold Copy THREDDS server if deploy: # download the data for the deployment flort = load_gc_thredds(site, node, sensor, method, stream, ('.*deployment%04d.*FLORT.*\\.nc$' % deploy)) # check to see if we downloaded any data if not flort: exit_text = ( 'Data unavailable for %s-%s-%s, %s, %s, deployment %d.' % (site, node, sensor, method, stream, deploy)) raise SystemExit(exit_text) else: # otherwise, request the data for download from OOINet via the M2M API using the specified dates r = m2m_request(site, node, sensor, method, stream, start, stop) if not r: exit_text = ( 'Request failed for %s-%s-%s, %s, %s, from %s to %s.' % (site, node, sensor, method, stream, start, stop)) raise SystemExit(exit_text) # Valid M2M request, start downloading the data flort = m2m_collect(r, '.*FLORT.*\\.nc$') # check to see if we downloaded any data if not flort: exit_text = ( 'Data unavailable for %s-%s-%s, %s, %s, from %s to %s.' % (site, node, sensor, method, stream, start, stop)) raise SystemExit(exit_text) # clean-up and reorganize the data if node == 'SP001': # this FLORT is part of a CSPP flort = flort_cspp(flort) elif node == 'WFP01': # this FLORT is part of a Wire-Following Profiler flort = flort_wfp(flort) elif node == 'SBD17': # this FLORT is connected to the CTDBP on an EA Inshore Surface Mooring flort = flort_instrument(flort) if not flort: # there was no data after removing all the 0's sys.exit() else: # this FLORT is stand-alone on one of the moorings flort = flort_datalogger(flort, burst) vocab = get_vocabulary(site, node, sensor)[0] flort = update_dataset(flort, vocab['maxdepth']) # save the data to disk out_file = os.path.abspath(args.outfile) if not os.path.exists(os.path.dirname(out_file)): os.makedirs(os.path.dirname(out_file)) flort.to_netcdf(out_file, mode='w', format='NETCDF4', engine='h5netcdf', encoding=ENCODINGS)
def main(argv=None): # setup the input arguments args = inputs(argv) site = args.site node = args.node sensor = args.sensor method = args.method stream = args.stream deploy = args.deploy start = args.start stop = args.stop burst = args.burst # determine the start and stop times for the data request based on either the deployment number or user entered # beginning and ending dates. if not deploy or (start and stop): return SyntaxError( 'You must specify either a deployment number or beginning and end dates of interest.' ) else: if deploy: # Determine start and end dates based on the deployment number start, stop = get_deployment_dates(site, node, sensor, deploy) if not start or not stop: exit_text = ( 'Deployment dates are unavailable for %s-%s-%s, deployment %02d.' % (site, node, sensor, deploy)) raise SystemExit(exit_text) # Request the data for download r = m2m_request(site, node, sensor, method, stream, start, stop) if not r: exit_text = ('Request failed for %s-%s-%s. Check request.' % (site, node, sensor)) raise SystemExit(exit_text) # Valid request, start downloading the data if re.match(r'.*_air.*', stream): if deploy: pco2a = m2m_collect( r, ('.*deployment%04d.*PCO2A.*air.*\\.nc$' % deploy)) else: pco2a = m2m_collect(r, '.*PCO2A.*air.*\\.nc$') nc_group = 'air' else: if deploy: pco2a = m2m_collect( r, ('.*deployment%04d.*PCO2A.*water.*\\.nc$' % deploy)) else: pco2a = m2m_collect(r, '.*PCO2A.*water.*\\.nc$') nc_group = 'water' if not pco2a: exit_text = ('Data unavailable for %s-%s-%s. Check request.' % (site, node, sensor)) raise SystemExit(exit_text) # clean-up and reorganize pco2a = pco2a_datalogger(pco2a, burst) vocab = get_vocabulary(site, node, sensor)[0] pco2a = update_dataset(pco2a, vocab['maxdepth']) # save the data to disk out_file = os.path.abspath(args.outfile) if not os.path.exists(os.path.dirname(out_file)): os.makedirs(os.path.dirname(out_file)) if os.path.isfile(out_file): pco2a.to_netcdf(out_file, mode='a', format='NETCDF4', engine='h5netcdf', encoding=ENCODINGS, group=nc_group) else: pco2a.to_netcdf(out_file, mode='w', format='NETCDF4', engine='h5netcdf', encoding=ENCODINGS, group=nc_group)
def main(argv=None): # setup the input arguments args = inputs(argv) site = args.site node = args.node sensor = args.sensor method = args.method stream = args.stream deploy = args.deploy start = args.start stop = args.stop # check if we are specifying a deployment or a specific date and time range if not deploy or (start and stop): return SyntaxError( 'You must specify either a deployment number or beginning and end dates of interest.' ) # if we are specifying a deployment number, then get the data from the Gold Copy THREDDS server if deploy: # download the data for the deployment pco2w = load_gc_thredds( site, node, sensor, method, stream, ('^(?!.*blank).*deployment%04d.*PCO2W.*\\.nc$' % deploy)) # check to see if we downloaded any data if not pco2w: exit_text = ( 'Data unavailable for %s-%s-%s, %s, %s, deployment %d.' % (site, node, sensor, method, stream, deploy)) raise SystemExit(exit_text) else: # otherwise, request the data for download from OOINet via the M2M API using the specified dates r = m2m_request(site, node, sensor, method, stream, start, stop) if not r: exit_text = ( 'Request failed for %s-%s-%s, %s, %s, from %s to %s.' % (site, node, sensor, method, stream, start, stop)) raise SystemExit(exit_text) # Valid M2M request, start downloading the data pco2w = m2m_collect(r, '^(?!.*blank).*PCO2W.*\\.nc$') # check to see if we downloaded any data if not pco2w: exit_text = ( 'Data unavailable for %s-%s-%s, %s, %s, from %s to %s.' % (site, node, sensor, method, stream, start, stop)) raise SystemExit(exit_text) # clean-up and reorganize if method in ['telemetered', 'recovered_host']: pco2w = pco2w_datalogger(pco2w) else: pco2w = pco2w_instrument(pco2w) vocab = get_vocabulary(site, node, sensor)[0] pco2w = update_dataset(pco2w, vocab['maxdepth']) # save the data to disk out_file = os.path.abspath(args.outfile) if not os.path.exists(os.path.dirname(out_file)): os.makedirs(os.path.dirname(out_file)) pco2w.to_netcdf(out_file, mode='w', format='NETCDF4', engine='h5netcdf', encoding=ENCODINGS)
def main(argv=None): args = inputs(argv) site = args.site node = args.node sensor = args.sensor method = args.method stream = args.stream deploy = args.deploy start = args.start stop = args.stop burst = args.burst # determine the start and stop times for the data request based on either the deployment number or user entered # beginning and ending dates. if not deploy or (start and stop): return SyntaxError( 'You must specify either a deployment number or beginning and end dates of interest.' ) else: if deploy: # Determine start and end dates based on the deployment number start, stop = get_deployment_dates(site, node, sensor, deploy) if not start or not stop: exit_text = ( 'Deployment dates are unavailable for %s-%s-%s, deployment %02d.' % (site, node, sensor, deploy)) raise SystemExit(exit_text) if stream not in ['suna_dcl_recovered']: exit_text = ( 'Currently the only stream supported is suna_dcl_recovered, you requested %s.' % stream) raise SystemExit(exit_text) # Request the data for download r = m2m_request(site, node, sensor, method, stream, start, stop) if not r: exit_text = ('Request failed for %s-%s-%s. Check request.' % (site, node, sensor)) raise SystemExit(exit_text) # Valid request, start downloading the data if deploy: nutnr = m2m_collect(r, ('.*deployment%04d.*NUTNR.*\\.nc$' % deploy)) else: nutnr = m2m_collect(r, '.*NUTNR.*\\.nc$') if not nutnr: exit_text = ('Data unavailable for %s-%s-%s. Check request.' % (site, node, sensor)) raise SystemExit(exit_text) # clean-up and reorganize nutnr = nutnr_datalogger(nutnr, burst) vocab = get_vocabulary(site, node, sensor)[0] nutnr = update_dataset(nutnr, vocab['maxdepth']) # save the data to disk out_file = os.path.abspath(args.outfile) if not os.path.exists(os.path.dirname(out_file)): os.makedirs(os.path.dirname(out_file)) nutnr.to_netcdf(out_file, mode='w', format='NETCDF4', engine='h5netcdf', encoding=ENCODINGS)