def main(src_h5, dst_h5, var_attrs_path, hub_height, chunk_size, weeks_per_chunk, overwrite, meta, process_size, check_dset_attrs, resolution, log_file, verbose): """ RechunkH5 CLI entry point """ if verbose: log_level = 'DEBUG' else: log_level = 'INFO' if log_file is not None: log_dir = os.path.dirname(log_file) if not os.path.exists(log_dir): os.makedirs(log_dir) init_logger('rex', log_file=log_file, log_level=log_level) dst_dir = os.path.dirname(dst_h5) if not os.path.exists(dst_dir): os.makedirs(dst_dir) RechunkH5.run(src_h5, dst_h5, var_attrs=var_attrs_path, hub_height=hub_height, chunk_size=chunk_size, weeks_per_chunk=weeks_per_chunk, overwrite=overwrite, meta=meta, process_size=process_size, check_dset_attrs=check_dset_attrs, resolution=resolution)
def test_profiles_means(): """ Test adding means to pre-collected profiles """ init_logger('reV.handlers.collection') h5_file = os.path.join(TEMP_DIR, 'cf.h5') Collector.collect(h5_file, H5_DIR, POINTS_PATH, 'cf_profile', dset_out=None, file_prefix='peregrine_2012') Collector.add_dataset(h5_file, H5_DIR, 'cf_mean', dset_out=None, file_prefix='peregrine_2012') with h5py.File(h5_file, 'r') as f: assert 'cf_profile' in f assert 'cf_mean' in f data = f['cf_profile'][...] node_file = os.path.join(H5_DIR, 'peregrine_2012_node01_x001.h5') with h5py.File(node_file, 'r') as f: source_data = f['cf_profile'][...] assert np.allclose(source_data, data[:, -1 * source_data.shape[1]:]) if PURGE_OUT: os.remove(h5_file)
def test_collection(): """ Test collection on 'cf_profile' ensuring output array is correct """ init_logger('reV.handlers.collection') profiles = manual_collect(H5_DIR, 'peregrine_2012', 'cf_profile') h5_file = os.path.join(TEMP_DIR, 'collection.h5') Collector.collect(h5_file, H5_DIR, POINTS_PATH, 'cf_profile', dset_out=None, file_prefix='peregrine_2012') with h5py.File(h5_file, 'r') as f: cf_profiles = f['cf_profile'][...] diff = np.mean(np.abs(profiles - cf_profiles)) msg = "Arrays differ by {:.4f}".format(diff) assert np.allclose(profiles, cf_profiles), msg source_file = os.path.join(H5_DIR, "peregrine_2012_node00_x000.h5") with h5py.File(source_file, 'r') as f_s: def check_attrs(name, object): object_s = f_s[name] for k, v in object.attrs.items(): v_s = object_s.attrs[k] assert v == v_s with h5py.File(h5_file, 'r') as f: f.visititems(check_attrs) if PURGE_OUT: os.remove(h5_file)
def reV_h5(ctx, h5_file, out_dir, sub_dir, dsets, group, process_size, max_workers, plot_type, cmap, log_file, verbose, terminal): """ Summarize and plot data for reV h5_file """ name = ctx.obj['NAME'] if any([verbose, ctx.obj['VERBOSE']]): log_level = 'DEBUG' else: log_level = 'INFO' init_logger('reV', log_file=log_file, log_level=log_level) qa_dir = out_dir if sub_dir is not None: qa_dir = os.path.join(out_dir, sub_dir) QaQc.h5(h5_file, qa_dir, dsets=dsets, group=group, process_size=process_size, max_workers=max_workers, plot_type=plot_type, cmap=cmap) if terminal: status = { 'dirout': out_dir, 'job_status': 'successful', 'finput': h5_file } Status.make_job_file(out_dir, 'qa-qc', name, status)
def test_low_mem_collect(): """Test memory limited multi chunk collection""" init_logger('reV.handlers.collection', log_level='DEBUG') h5_file = os.path.join(TEMP_DIR, 'cf.h5') Collector.collect(h5_file, H5_DIR, POINTS_PATH, 'cf_profile', dset_out=None, file_prefix='peregrine_2012', mem_util_lim=0.00002) with h5py.File(h5_file, 'r') as f: assert 'cf_profile' in f data = f['cf_profile'][...] node_file = os.path.join(H5_DIR, 'peregrine_2012_node01_x001.h5') with h5py.File(node_file, 'r') as f: source_data = f['cf_profile'][...] assert np.allclose(source_data, data[:, -1 * source_data.shape[1]:]) if PURGE_OUT: os.remove(h5_file)
def supply_curve(ctx, sc_table, out_dir, sub_dir, columns, plot_type, cmap, lcoe, log_file, verbose, terminal): """ Summarize and plot reV Supply Curve data """ name = ctx.obj['NAME'] if any([verbose, ctx.obj['VERBOSE']]): log_level = 'DEBUG' else: log_level = 'INFO' init_logger('reV', log_file=log_file, log_level=log_level) qa_dir = out_dir if sub_dir is not None: qa_dir = os.path.join(out_dir, sub_dir) QaQc.supply_curve(sc_table, qa_dir, columns=columns, lcoe=lcoe, plot_type=plot_type, cmap=cmap) if terminal: status = { 'dirout': out_dir, 'job_status': 'successful', 'finput': sc_table } Status.make_job_file(out_dir, 'qa-qc', name, status)
def main(ctx, wave_h5, out_dir, log_file, verbose): """ WaveX Command Line Interface """ ctx.ensure_object(dict) if not os.path.exists(out_dir): os.makedirs(out_dir) ctx.obj['H5'] = wave_h5 ctx.obj['OUT_DIR'] = out_dir ctx.obj['CLS_KWARGS'] = {} ctx.obj['CLS'] = WaveX _, hsds = check_res_file(wave_h5) if hsds: ctx.obj['CLS_KWARGS']['hsds'] = hsds else: assert os.path.exists(wave_h5) if verbose: log_level = 'DEBUG' else: log_level = 'INFO' if log_file is not None: log_dir = os.path.dirname(log_file) if not os.path.exists(log_dir): os.makedirs(log_dir) init_logger('rex', log_file=log_file, log_level=log_level) logger.info('Extracting wave data from {}'.format(wave_h5)) logger.info('Outputs to be stored in: {}'.format(out_dir))
def main(combined_h5, source_h5, axis, overwrite, process_size, log_file, verbose): """ CombineH5 CLI entry point """ if verbose: log_level = 'DEBUG' else: log_level = 'INFO' if log_file is not None: log_dir = os.path.dirname(log_file) if not os.path.exists(log_dir): os.makedirs(log_dir) init_logger('rex', log_file=log_file, log_level=log_level) dst_dir = os.path.dirname(combined_h5) if not os.path.exists(dst_dir): os.makedirs(dst_dir) CombineH5.run(combined_h5, *source_h5, axis=axis, overwrite=overwrite, process_size=process_size)
def main(ctx, resource_path, out_dir, years, hsds, res_cls, log_file, verbose): """ Multi-year ResourceX Command Line Interface """ ctx.ensure_object(dict) ctx.obj['H5'] = resource_path ctx.obj['OUT_DIR'] = out_dir ctx.obj['CLS_KWARGS'] = {'years': years, 'hsds': hsds} if res_cls == 'Resource': ctx.obj['CLS'] = MultiYearResourceX elif res_cls == 'NSRDB': ctx.obj['CLS'] = MultiYearNSRDBX elif res_cls == 'Wind': ctx.obj['CLS'] = MultiYearWindX elif res_cls == 'Wave': ctx.obj['CLS'] = MultiYearWaveX if verbose: log_level = 'DEBUG' else: log_level = 'INFO' if log_file is not None: log_dir = os.path.dirname(log_file) if not os.path.exists(log_dir): os.makedirs(log_dir) init_logger('rex', log_file=log_file, log_level=log_level) logger.info('Extracting Resource data from {}'.format(resource_path)) logger.info('Outputs to be stored in: {}'.format(out_dir))
def __init__(self, config, verbose=False): """ Parameters ---------- config : str File path to config json or csv (str). verbose : bool Flag to turn on debug logging. """ self._job_tags = None self._config = BatchConfig(config) self._base_dir = self._config.config_dir os.chdir(self._base_dir) if 'logging' in self._config: logging_kwargs = self._config.get('logging', {}) if verbose: logging_kwargs['log_level'] = 'DEBUG' init_logger('reV.batch', **logging_kwargs) x = self._parse_config(self._config) self._arg_combs, self._file_sets, self._set_tags = x logger.info('Batch job initialized with {} sub jobs.'.format( len(self._arg_combs)))
def summarize(ctx, out_dir, log_file, verbose): """ Summarize reV data """ ctx.obj['OUT_DIR'] = out_dir if any([verbose, ctx.obj['VERBOSE']]): log_level = 'DEBUG' else: log_level = 'INFO' init_logger('reV', log_file=log_file, log_level=log_level)
def main(ctx, fpath, res_file, sam_file, verbose): """reV ProjectPoints generator""" ctx.ensure_object(dict) ctx.obj['FPATH'] = fpath ctx.obj['RES_FILE'] = res_file ctx.obj['SAM_FILE'] = sam_file ctx.obj['VERBOSE'] = verbose if verbose: log_level = 'DEBUG' else: log_level = 'INFO' init_logger('reV.config.project_points', log_level=log_level)
def test_collect_means(): """ Test means collection: """ init_logger('reV.handlers.collection') h5_file = os.path.join(TEMP_DIR, 'cf_means.h5') Collector.collect(h5_file, H5_DIR, POINTS_PATH, 'cf_mean', dset_out=None, file_prefix='peregrine_2012') if PURGE_OUT: os.remove(h5_file)
def main(ctx, src_h5, dst_h5, var_attrs_path, version, meta, process_size, check_dset_attrs, log_file, verbose): """ RechunkH5 CLI entry point """ ctx.ensure_object(dict) if verbose: log_level = 'DEBUG' else: log_level = 'INFO' if log_file is not None: log_dir = os.path.dirname(log_file) if not os.path.exists(log_dir): os.makedirs(log_dir) logger = init_logger('rex.rechunk_h5.rechunk_h5', log_file=log_file, log_level=log_level) ctx.obj['LOGGER'] = logger dst_dir = os.path.dirname(dst_h5) if not os.path.exists(dst_dir): os.makedirs(dst_dir) RechunkH5.run(src_h5, dst_h5, var_attrs_path, version=version, meta=meta, process_size=process_size, check_dset_attrs=check_dset_attrs)
def __init__(self, pipeline, monitor=True, verbose=False): """ Parameters ---------- pipeline : str | dict Pipeline config file path or dictionary. monitor : bool Flag to perform continuous monitoring of the pipeline. verbose : bool Flag to submit pipeline steps with -v flag for debug logging """ self.monitor = monitor self.verbose = verbose self._config = PipelineConfig(pipeline) self._run_list = self._config.pipeline_steps self._init_status() # init logger for pipeline module if requested in input config if 'logging' in self._config: init_logger('reV.pipeline', **self._config['logging'])
def main(ctx, solar_h5, out_dir, log_file, verbose): """ NSRDBX Command Line Interface """ ctx.ensure_object(dict) if not os.path.exists(out_dir): os.makedirs(out_dir) ctx.obj['H5'] = solar_h5 ctx.obj['OUT_DIR'] = out_dir ctx.obj['CLS_KWARGS'] = {} multi_h5_res, hsds = check_res_file(solar_h5) name = os.path.splitext(os.path.basename(solar_h5))[0] if 'nsrdb' in name: ctx.obj['CLS'] = NSRDBX else: ctx.obj['CLS'] = SolarX if multi_h5_res and not hsds: assert os.path.exists(os.path.dirname(solar_h5)) ctx.obj['CLS'] = MultiFileNSRDBX elif hsds: ctx.obj['CLS_KWARGS']['hsds'] = hsds else: assert os.path.exists(solar_h5) if verbose: log_level = 'DEBUG' else: log_level = 'INFO' if log_file is not None: log_dir = os.path.dirname(log_file) if not os.path.exists(log_dir): os.makedirs(log_dir) init_logger('rex', log_file=log_file, log_level=log_level) logger.info('Extracting solar data from {}'.format(solar_h5)) logger.info('Outputs to be stored in: {}'.format(out_dir))
def main(ctx, domain, out_dir, years, buoy, eagle, log_file, verbose): """ US Wave Command Line Interface """ ctx.ensure_object(dict) if eagle: path = '/datasets/US_wave/v1.0.0' hsds = False else: path = '/nrel/US_wave' hsds = True if buoy: path = os.path.join(path, 'virtual_buoy', domain) else: path = os.path.join(path, domain) ctx.obj['H5'] = path if not os.path.exists(out_dir): os.makedirs(out_dir) ctx.obj['OUT_DIR'] = out_dir ctx.obj['CLS_KWARGS'] = {'years': years, 'hsds': hsds} ctx.obj['CLS'] = MultiYearWaveX if verbose: log_level = 'DEBUG' else: log_level = 'INFO' if log_file is not None: log_dir = os.path.dirname(log_file) if not os.path.exists(log_dir): os.makedirs(log_dir) init_logger('rex', log_file=log_file, log_level=log_level) logger.info('Extracting US wave data for {} from {}'.format(domain, path)) logger.info('Outputs to be stored in: {}'.format(out_dir))
def exclusions(ctx, excl_fpath, out_dir, sub_dir, excl_dict, area_filter_kernel, min_area, plot_type, cmap, plot_step, log_file, verbose, terminal): """ Extract and plot reV exclusions mask """ name = ctx.obj['NAME'] if any([verbose, ctx.obj['VERBOSE']]): log_level = 'DEBUG' else: log_level = 'INFO' init_logger('reV', log_file=log_file, log_level=log_level) qa_dir = out_dir if sub_dir is not None: qa_dir = os.path.join(out_dir, sub_dir) if isinstance(excl_dict, str): excl_dict = dict_str_load(excl_dict) QaQc.exclusions_mask(excl_fpath, qa_dir, layers_dict=excl_dict, min_area=min_area, kernel=area_filter_kernel, plot_type=plot_type, cmap=cmap, plot_step=plot_step) if terminal: status = { 'dirout': out_dir, 'job_status': 'successful', 'finput': excl_fpath } Status.make_job_file(out_dir, 'qa-qc', name, status)
def test_means_lcoe(): """ Test adding means to pre-collected profiles """ init_logger('reV.handlers.collection') h5_file = os.path.join(TEMP_DIR, 'cf_lcoe.h5') Collector.collect(h5_file, H5_DIR, POINTS_PATH, 'cf_mean', dset_out=None, file_prefix='peregrine_2012') Collector.add_dataset(h5_file, H5_DIR, 'lcoe_fcr', dset_out=None, file_prefix='peregrine_2012') with h5py.File(h5_file, 'r') as f: assert 'cf_mean' in f assert 'lcoe_fcr' in f if PURGE_OUT: os.remove(h5_file)
def test_eagle(year): """Gen PV CF profiles with write to disk and compare against rev1.""" res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) sam_files = TESTDATADIR + '/SAM/naris_pv_1axis_inv13.json' rev2_out_dir = os.path.join(TESTDATADIR, 'ri_pv_reV2') rev2_out = 'gen_ri_pv_smart_{}.h5'.format(year) if not os.path.exists(rev2_out_dir): os.mkdir(rev2_out_dir) name = 'etest' points = slice(0, 100) verbose = True log_level = 'DEBUG' log_file = os.path.join(rev2_out_dir, '{}.log'.format(name)) modules = [__name__, 'reV.utilities', 'reV.generation'] for mod in modules: init_logger(mod, log_level=log_level, log_file=log_file) cmd = get_node_cmd(name=name, tech='pvwattsv5', points=points, points_range=None, sam_files=sam_files, res_file=res_file, sites_per_worker=None, max_workers=None, fout=rev2_out, dirout=rev2_out_dir, logdir=rev2_out_dir, output_request=('cf_profile', 'cf_mean'), verbose=verbose) # create and submit the SLURM job slurm = SLURM(cmd, alloc='rev', memory=96, walltime=0.1, name=name, stdout_path=rev2_out_dir) while True: status = slurm.check_status(name, var='name') if status == 'CG': break else: time.sleep(5) # get reV 2.0 generation profiles from disk flist = os.listdir(rev2_out_dir) for fname in flist: if '.h5' in fname: if rev2_out.strip('.h5') in fname: full_f = os.path.join(rev2_out_dir, fname) with Outputs(full_f, 'r') as cf: rev2_profiles = cf['cf_profile'] break # get reV 1.0 generation profiles rev1_profiles = get_r1_profiles(year=year) rev1_profiles = rev1_profiles[:, points] result = np.allclose(rev1_profiles, rev2_profiles, rtol=RTOL, atol=ATOL) if result: # remove output files if test passes. flist = os.listdir(rev2_out_dir) for fname in flist: os.remove(os.path.join(rev2_out_dir, fname)) assert result is True
from reV import TESTDATADIR from rex.utilities.loggers import init_logger TEMP_DIR = os.path.join(TESTDATADIR, 'ri_gen_collect') H5_DIR = os.path.join(TESTDATADIR, 'gen_out') YEARS = [2012, 2013] H5_FILES = [ os.path.join(H5_DIR, 'gen_ri_pv_{}_x000.h5'.format(year)) for year in YEARS ] PURGE_OUT = True if not os.path.exists(TEMP_DIR): os.makedirs(TEMP_DIR) logger = init_logger('reV.handlers.multi_year', log_level='DEBUG') def manual_means(h5_files, dset): """ Manually calculate means for dset in .h5 files Parameters ---------- h5_files : list List of .h5 files to compute means from dset : str Dataset to collect Results -------