def supply_curve(ctx, sc_table, out_dir, sub_dir, columns, plot_type, cmap, lcoe, log_file, verbose, terminal): """ Summarize and plot reV Supply Curve data """ name = ctx.obj['NAME'] if any([verbose, ctx.obj['VERBOSE']]): log_level = 'DEBUG' else: log_level = 'INFO' init_logger('reV', log_file=log_file, log_level=log_level) qa_dir = out_dir if sub_dir is not None: qa_dir = os.path.join(out_dir, sub_dir) QaQc.supply_curve(sc_table, qa_dir, columns=columns, lcoe=lcoe, plot_type=plot_type, cmap=cmap) if terminal: status = { 'dirout': out_dir, 'job_status': 'successful', 'finput': sc_table } Status.make_job_file(out_dir, 'qa-qc', name, status)
def reV_h5(ctx, h5_file, out_dir, sub_dir, dsets, group, process_size, max_workers, plot_type, cmap, log_file, verbose, terminal): """ Summarize and plot data for reV h5_file """ name = ctx.obj['NAME'] if any([verbose, ctx.obj['VERBOSE']]): log_level = 'DEBUG' else: log_level = 'INFO' init_logger('reV', log_file=log_file, log_level=log_level) qa_dir = out_dir if sub_dir is not None: qa_dir = os.path.join(out_dir, sub_dir) QaQc.h5(h5_file, qa_dir, dsets=dsets, group=group, process_size=process_size, max_workers=max_workers, plot_type=plot_type, cmap=cmap) if terminal: status = { 'dirout': out_dir, 'job_status': 'successful', 'finput': h5_file } Status.make_job_file(out_dir, 'qa-qc', name, status)
def test_make_file(): """Test file creation and reading""" purge() Status.make_job_file(STATUS_DIR, 'generation', 'test1', TEST_1_ATTRS_1) status = Status.retrieve_job_status(STATUS_DIR, 'generation', 'test1') msg = 'Failed, status is "{}"'.format(status) assert status == 'R', msg purge()
def direct(ctx, gen_fpath, rev_summary, reg_cols, cf_dset, rep_method, err_method, weight, n_profiles, out_dir, log_dir, max_workers, aggregate_profiles, verbose): """reV representative profiles CLI.""" name = ctx.obj['NAME'] ctx.obj['GEN_FPATH'] = gen_fpath ctx.obj['REV_SUMMARY'] = rev_summary ctx.obj['REG_COLS'] = reg_cols ctx.obj['CF_DSET'] = cf_dset ctx.obj['REP_METHOD'] = rep_method ctx.obj['ERR_METHOD'] = err_method ctx.obj['WEIGHT'] = weight ctx.obj['N_PROFILES'] = n_profiles ctx.obj['OUT_DIR'] = out_dir ctx.obj['LOG_DIR'] = log_dir ctx.obj['MAX_WORKERS'] = max_workers ctx.obj['AGGREGATE_PROFILES'] = aggregate_profiles ctx.obj['VERBOSE'] = verbose if ctx.invoked_subcommand is None: t0 = time.time() init_mult(name, log_dir, modules=['reV', 'rex'], verbose=verbose) fn_out = '{}.h5'.format(name) fout = os.path.join(out_dir, fn_out) if aggregate_profiles: AggregatedRepProfiles.run(gen_fpath, rev_summary, cf_dset=cf_dset, weight=weight, fout=fout, max_workers=max_workers) else: RepProfiles.run(gen_fpath, rev_summary, reg_cols, cf_dset=cf_dset, rep_method=rep_method, err_method=err_method, weight=weight, fout=fout, n_profiles=n_profiles, max_workers=max_workers) runtime = (time.time() - t0) / 60 logger.info('reV representative profiles complete. ' 'Time elapsed: {:.2f} min. Target output dir: {}'.format( runtime, out_dir)) status = { 'dirout': out_dir, 'fout': fn_out, 'job_status': 'successful', 'runtime': runtime, 'finput': [gen_fpath, rev_summary] } Status.make_job_file(out_dir, 'rep-profiles', name, status)
def multi_year_groups(ctx, group_params, verbose): """Run multi year collection and means for multiple groups.""" name = ctx.obj['NAME'] my_file = ctx.obj['MY_FILE'] verbose = any([verbose, ctx.obj['VERBOSE']]) # initialize loggers for multiple modules log_dir = os.path.dirname(my_file) init_mult(name, log_dir, modules=[__name__, 'reV.handlers.multi_year'], verbose=verbose, node=True) for key, val in ctx.obj.items(): logger.debug('ctx var passed to collection method: "{}" : "{}" ' 'with type "{}"'.format(key, val, type(val))) logger.info('Multi-year collection is being run with job name "{}". ' 'Target output path is: {}'.format(name, my_file)) ts = time.time() for group_name, group in json.loads(group_params).items(): logger.info('- Collecting datasets "{}" from "{}" into "{}/"'.format( group['dsets'], group['source_files'], group_name)) t0 = time.time() for dset in group['dsets']: if MultiYear.is_profile(group['source_files'], dset): MultiYear.collect_profiles(my_file, group['source_files'], dset, group=group['group']) else: MultiYear.collect_means(my_file, group['source_files'], dset, group=group['group']) runtime = (time.time() - t0) / 60 logger.info('- {} collection completed in: {:.2f} min.'.format( group_name, runtime)) runtime = (time.time() - ts) / 60 logger.info( 'Multi-year collection completed in : {:.2f} min.'.format(runtime)) # add job to reV status file. status = { 'dirout': os.path.dirname(my_file), 'fout': os.path.basename(my_file), 'job_status': 'successful', 'runtime': runtime } Status.make_job_file(os.path.dirname(my_file), 'multi-year', name, status)
def test_file_collection(): """Test file creation and collection""" purge() Status.make_job_file(STATUS_DIR, 'generation', 'test1', TEST_1_ATTRS_1) Status.make_job_file(STATUS_DIR, 'generation', 'test2', TEST_2_ATTRS_1) Status.update(STATUS_DIR) with open(os.path.join(STATUS_DIR, 'rev_status.json'), 'r') as f: data = json.load(f) assert str(TEST_1_ATTRS_1) in str(data) assert str(TEST_2_ATTRS_1) in str(data) purge()
def multi_year(ctx, source_files, group, dsets, pass_through_dsets, verbose): """Run multi year collection and means on local worker.""" name = ctx.obj['NAME'] my_file = ctx.obj['MY_FILE'] verbose = any([verbose, ctx.obj['VERBOSE']]) # initialize loggers for multiple modules log_dir = os.path.dirname(my_file) init_mult(name, log_dir, modules=[__name__, 'reV.handlers.multi_year'], verbose=verbose, node=True) for key, val in ctx.obj.items(): logger.debug('ctx var passed to collection method: "{}" : "{}" ' 'with type "{}"'.format(key, val, type(val))) logger.info('Multi-year collection is being run for "{}" ' 'with job name "{}" on {}. Target output path is: {}'.format( dsets, name, source_files, my_file)) t0 = time.time() for dset in dsets: if MultiYear.is_profile(source_files, dset): MultiYear.collect_profiles(my_file, source_files, dset, group=group) else: MultiYear.collect_means(my_file, source_files, dset, group=group) if pass_through_dsets is not None: for dset in pass_through_dsets: MultiYear.pass_through(my_file, source_files, dset, group=group) runtime = (time.time() - t0) / 60 logger.info( 'Multi-year collection completed in: {:.2f} min.'.format(runtime)) # add job to reV status file. status = { 'dirout': os.path.dirname(my_file), 'fout': os.path.basename(my_file), 'job_status': 'successful', 'runtime': runtime, 'finput': source_files } Status.make_job_file(os.path.dirname(my_file), 'multi-year', name, status)
def direct(ctx, gen_fpath, offshore_fpath, points, sam_files, log_dir, verbose): """Main entry point to run offshore wind aggregation""" name = ctx.obj['NAME'] ctx.obj['GEN_FPATH'] = gen_fpath ctx.obj['OFFSHORE_FPATH'] = offshore_fpath ctx.obj['POINTS'] = points ctx.obj['SAM_FILES'] = sam_files ctx.obj['OUT_DIR'] = os.path.dirname(gen_fpath) ctx.obj['LOG_DIR'] = log_dir ctx.obj['VERBOSE'] = verbose if ctx.invoked_subcommand is None: t0 = time.time() init_mult(name, log_dir, modules=[__name__, 'reV.offshore', 'reV.handlers', 'rex'], verbose=verbose, node=True) fpath_out = gen_fpath.replace('.h5', '_offshore.h5') try: Offshore.run(gen_fpath, offshore_fpath, points, sam_files, fpath_out=fpath_out) except Exception as e: logger.exception('Offshore module failed, received the ' 'following exception:\n{}'.format(e)) raise e runtime = (time.time() - t0) / 60 status = { 'dirout': os.path.dirname(fpath_out), 'fout': os.path.basename(fpath_out), 'job_status': 'successful', 'runtime': runtime, 'finput': gen_fpath } Status.make_job_file(os.path.dirname(fpath_out), 'offshore', name, status)
def exclusions(ctx, excl_fpath, out_dir, sub_dir, excl_dict, area_filter_kernel, min_area, plot_type, cmap, plot_step, log_file, verbose, terminal): """ Extract and plot reV exclusions mask """ name = ctx.obj['NAME'] if any([verbose, ctx.obj['VERBOSE']]): log_level = 'DEBUG' else: log_level = 'INFO' init_logger('reV', log_file=log_file, log_level=log_level) qa_dir = out_dir if sub_dir is not None: qa_dir = os.path.join(out_dir, sub_dir) if isinstance(excl_dict, str): excl_dict = dict_str_load(excl_dict) QaQc.exclusions_mask(excl_fpath, qa_dir, layers_dict=excl_dict, min_area=min_area, kernel=area_filter_kernel, plot_type=plot_type, cmap=cmap, plot_step=plot_step) if terminal: status = { 'dirout': out_dir, 'job_status': 'successful', 'finput': excl_fpath } Status.make_job_file(out_dir, 'qa-qc', name, status)
def direct(ctx, sc_points, trans_table, fixed_charge_rate, sc_features, transmission_costs, sort_on, offshore_trans_table, wind_dirs, n_dirs, downwind, offshore_compete, max_workers, out_dir, log_dir, simple, line_limited, verbose): """reV Supply Curve CLI.""" name = ctx.obj['NAME'] ctx.obj['SC_POINTS'] = sc_points ctx.obj['TRANS_TABLE'] = trans_table ctx.obj['FIXED_CHARGE_RATE'] = fixed_charge_rate ctx.obj['SC_FEATURES'] = sc_features ctx.obj['TRANSMISSION_COSTS'] = transmission_costs ctx.obj['SORT_ON'] = sort_on ctx.obj['OFFSHORE_TRANS_TABLE'] = offshore_trans_table ctx.obj['WIND_DIRS'] = wind_dirs ctx.obj['N_DIRS'] = n_dirs ctx.obj['DOWNWIND'] = downwind ctx.obj['MAX_WORKERS'] = max_workers ctx.obj['OFFSHORE_COMPETE'] = offshore_compete ctx.obj['OUT_DIR'] = out_dir ctx.obj['LOG_DIR'] = log_dir ctx.obj['SIMPLE'] = simple ctx.obj['LINE_LIMITED'] = line_limited ctx.obj['VERBOSE'] = verbose if ctx.invoked_subcommand is None: t0 = time.time() init_mult( name, log_dir, modules=[__name__, 'reV.supply_curve', 'reV.handlers', 'rex'], verbose=verbose) if isinstance(transmission_costs, str): transmission_costs = dict_str_load(transmission_costs) offshore_table = offshore_trans_table try: if simple: out = SupplyCurve.simple(sc_points, trans_table, fixed_charge_rate, sc_features=sc_features, transmission_costs=transmission_costs, sort_on=sort_on, wind_dirs=wind_dirs, n_dirs=n_dirs, downwind=downwind, max_workers=max_workers, offshore_trans_table=offshore_table, offshore_compete=offshore_compete) else: out = SupplyCurve.full(sc_points, trans_table, fixed_charge_rate, sc_features=sc_features, transmission_costs=transmission_costs, line_limited=line_limited, sort_on=sort_on, wind_dirs=wind_dirs, n_dirs=n_dirs, downwind=downwind, max_workers=max_workers, offshore_trans_table=offshore_table, offshore_compete=offshore_compete) except Exception as e: logger.exception('Supply curve compute failed. Received the ' 'following error:\n{}'.format(e)) raise e fn_out = '{}.csv'.format(name) fpath_out = os.path.join(out_dir, fn_out) out.to_csv(fpath_out, index=False) runtime = (time.time() - t0) / 60 logger.info('Supply curve complete. Time elapsed: {:.2f} min. ' 'Target output dir: {}'.format(runtime, out_dir)) finput = [sc_points, trans_table] if sc_features is not None: finput.append(sc_features) if transmission_costs is not None: finput.append(transmission_costs) # add job to reV status file. status = { 'dirout': out_dir, 'fout': fn_out, 'job_status': 'successful', 'runtime': runtime, 'finput': finput } Status.make_job_file(out_dir, 'supply-curve', name, status)
def collect(ctx, verbose): """Run collection on local worker.""" name = ctx.obj['NAME'] h5_file = ctx.obj['H5_FILE'] h5_dir = ctx.obj['H5_DIR'] project_points = ctx.obj['PROJECT_POINTS'] dsets = ctx.obj['DSETS'] file_prefix = ctx.obj['FILE_PREFIX'] log_dir = ctx.obj['LOG_DIR'] purge_chunks = ctx.obj['PURGE_CHUNKS'] verbose = any([verbose, ctx.obj['VERBOSE']]) # initialize loggers for multiple modules init_mult(name, log_dir, modules=[__name__, 'reV.handlers.collection'], verbose=verbose, node=True) for key, val in ctx.obj.items(): logger.debug('ctx var passed to collection method: "{}" : "{}" ' 'with type "{}"'.format(key, val, type(val))) logger.info('Collection is being run for "{}" with job name "{}" ' 'and collection dir: {}. Target output path is: {}'.format( dsets, name, h5_dir, h5_file)) t0 = time.time() Collector.collect(h5_file, h5_dir, project_points, dsets[0], file_prefix=file_prefix) if len(dsets) > 1: for dset_name in dsets[1:]: Collector.add_dataset(h5_file, h5_dir, dset_name, file_prefix=file_prefix) if purge_chunks: Collector.purge_chunks(h5_file, h5_dir, project_points, file_prefix=file_prefix) else: Collector.move_chunks(h5_file, h5_dir, project_points, file_prefix=file_prefix) runtime = (time.time() - t0) / 60 logger.info('Collection completed in: {:.2f} min.'.format(runtime)) # add job to reV status file. status = { 'dirout': os.path.dirname(h5_file), 'fout': os.path.basename(h5_file), 'job_status': 'successful', 'runtime': runtime, 'finput': os.path.join(h5_dir, '{}*.h5'.format(file_prefix)) } Status.make_job_file(os.path.dirname(h5_file), 'collect', name, status)
def local(ctx, max_workers, timeout, points_range, verbose): """Run econ on local worker(s).""" name = ctx.obj['NAME'] points = ctx.obj['POINTS'] sam_files = ctx.obj['SAM_FILES'] cf_file = ctx.obj['CF_FILE'] cf_year = ctx.obj['CF_YEAR'] site_data = ctx.obj['SITE_DATA'] sites_per_worker = ctx.obj['SITES_PER_WORKER'] fout = ctx.obj['FOUT'] dirout = ctx.obj['DIROUT'] logdir = ctx.obj['LOGDIR'] output_request = ctx.obj['OUTPUT_REQUEST'] append = ctx.obj['APPEND'] verbose = any([verbose, ctx.obj['VERBOSE']]) if append: fout = os.path.basename(cf_file) dirout = os.path.dirname(cf_file) # initialize loggers for multiple modules log_modules = [ __name__, 'reV.econ.econ', 'reV.generation', 'reV.config', 'reV.utilities', 'reV.SAM', 'reV.handlers', 'rex.utilities' ] init_mult(name, logdir, modules=log_modules, verbose=verbose, node=True) for key, val in ctx.obj.items(): logger.debug('ctx var passed to local method: "{}" : "{}" with type ' '"{}"'.format(key, val, type(val))) logger.info( 'Econ local is being run with with job name "{}" and ' 'generation results file: {}. Target output path is: {}'.format( name, cf_file, os.path.join(dirout, fout))) t0 = time.time() # Execute the Generation module with smart data flushing. Econ.reV_run(points=points, sam_files=sam_files, cf_file=cf_file, cf_year=cf_year, site_data=site_data, output_request=output_request, max_workers=max_workers, timeout=timeout, sites_per_worker=sites_per_worker, points_range=points_range, fout=fout, dirout=dirout, append=append) tmp_str = ' with points range {}'.format(points_range) runtime = (time.time() - t0) / 60 logger.info('Econ compute complete for project points "{0}"{1}. ' 'Time elapsed: {2:.2f} min. Target output dir: {3}'.format( points, tmp_str if points_range else '', runtime, dirout)) # add job to reV status file. status = { 'dirout': dirout, 'fout': fout, 'job_status': 'successful', 'runtime': runtime, 'finput': cf_file } Status.make_job_file(dirout, 'econ', name, status)
def direct(ctx, excl_fpath, gen_fpath, econ_fpath, res_fpath, tm_dset, excl_dict, check_excl_layers, res_class_dset, res_class_bins, cf_dset, lcoe_dset, h5_dsets, data_layers, resolution, excl_area, power_density, area_filter_kernel, min_area, friction_fpath, friction_dset, out_dir, log_dir, verbose): """reV Supply Curve Aggregation Summary CLI.""" name = ctx.obj['NAME'] ctx.obj['EXCL_FPATH'] = excl_fpath ctx.obj['GEN_FPATH'] = gen_fpath ctx.obj['ECON_FPATH'] = econ_fpath ctx.obj['RES_FPATH'] = res_fpath ctx.obj['TM_DSET'] = tm_dset ctx.obj['EXCL_DICT'] = excl_dict ctx.obj['CHECK_LAYERS'] = check_excl_layers ctx.obj['RES_CLASS_DSET'] = res_class_dset ctx.obj['RES_CLASS_BINS'] = res_class_bins ctx.obj['CF_DSET'] = cf_dset ctx.obj['LCOE_DSET'] = lcoe_dset ctx.obj['H5_DSETS'] = h5_dsets ctx.obj['DATA_LAYERS'] = data_layers ctx.obj['RESOLUTION'] = resolution ctx.obj['EXCL_AREA'] = excl_area ctx.obj['POWER_DENSITY'] = power_density ctx.obj['AREA_FILTER_KERNEL'] = area_filter_kernel ctx.obj['MIN_AREA'] = min_area ctx.obj['FRICTION_FPATH'] = friction_fpath ctx.obj['FRICTION_DSET'] = friction_dset ctx.obj['OUT_DIR'] = out_dir ctx.obj['LOG_DIR'] = log_dir ctx.obj['VERBOSE'] = verbose if ctx.invoked_subcommand is None: t0 = time.time() init_mult(name, log_dir, modules=[__name__, 'reV.supply_curve'], verbose=verbose) with h5py.File(excl_fpath, mode='r') as f: dsets = list(f) if tm_dset not in dsets: try: TechMapping.run(excl_fpath, res_fpath, tm_dset) except Exception as e: logger.exception('TechMapping process failed. Received the ' 'following error:\n{}'.format(e)) raise e if isinstance(excl_dict, str): excl_dict = dict_str_load(excl_dict) if isinstance(data_layers, str): data_layers = dict_str_load(data_layers) try: summary = SupplyCurveAggregation.summary( excl_fpath, gen_fpath, tm_dset, econ_fpath=econ_fpath, excl_dict=excl_dict, res_class_dset=res_class_dset, res_class_bins=res_class_bins, cf_dset=cf_dset, lcoe_dset=lcoe_dset, h5_dsets=h5_dsets, data_layers=data_layers, resolution=resolution, excl_area=excl_area, power_density=power_density, area_filter_kernel=area_filter_kernel, min_area=min_area, friction_fpath=friction_fpath, friction_dset=friction_dset, check_excl_layers=check_excl_layers) except Exception as e: logger.exception('Supply curve Aggregation failed. Received the ' 'following error:\n{}'.format(e)) raise e fn_out = '{}.csv'.format(name) fpath_out = os.path.join(out_dir, fn_out) summary.to_csv(fpath_out) runtime = (time.time() - t0) / 60 logger.info('Supply curve aggregation complete. ' 'Time elapsed: {:.2f} min. Target output dir: {}'.format( runtime, out_dir)) finput = [excl_fpath, gen_fpath] if res_fpath is not None: finput.append(res_fpath) # add job to reV status file. status = { 'dirout': out_dir, 'fout': fn_out, 'job_status': 'successful', 'runtime': runtime, 'finput': finput, 'excl_fpath': excl_fpath, 'excl_dict': excl_dict, 'area_filter_kernel': area_filter_kernel, 'min_area': min_area } Status.make_job_file(out_dir, 'supply-curve-aggregation', name, status)
def local(ctx, max_workers, timeout, points_range, verbose): """Run generation on local worker(s).""" name = ctx.obj['NAME'] tech = ctx.obj['TECH'] points = ctx.obj['POINTS'] sam_files = ctx.obj['SAM_FILES'] res_file = ctx.obj['RES_FILE'] sites_per_worker = ctx.obj['SITES_PER_WORKER'] fout = ctx.obj['FOUT'] dirout = ctx.obj['DIROUT'] logdir = ctx.obj['LOGDIR'] output_request = ctx.obj['OUTPUT_REQUEST'] site_data = ctx.obj['SITE_DATA'] mem_util_lim = ctx.obj['MEM_UTIL_LIM'] curtailment = ctx.obj['CURTAILMENT'] verbose = any([verbose, ctx.obj['VERBOSE']]) # initialize loggers for multiple modules init_mult(name, logdir, modules=['reV', 'rex'], verbose=verbose, node=True) for key, val in ctx.obj.items(): logger.debug('ctx var passed to local method: "{}" : "{}" with type ' '"{}"'.format(key, val, type(val))) logger.info('Gen local is being run with with job name "{}" and resource ' 'file: {}. Target output path is: {}'.format( name, res_file, os.path.join(dirout, fout))) t0 = time.time() points = _parse_points(ctx) # Execute the Generation module with smart data flushing. Gen.reV_run(tech=tech, points=points, sam_files=sam_files, res_file=res_file, site_data=site_data, output_request=output_request, curtailment=curtailment, max_workers=max_workers, sites_per_worker=sites_per_worker, points_range=points_range, fout=fout, dirout=dirout, mem_util_lim=mem_util_lim, timeout=timeout) tmp_str = ' with points range {}'.format(points_range) runtime = (time.time() - t0) / 60 logger.info('Gen compute complete for project points "{0}"{1}. ' 'Time elapsed: {2:.2f} min. Target output dir: {3}'.format( points, tmp_str if points_range else '', runtime, dirout)) # add job to reV status file. status = { 'dirout': dirout, 'fout': fout, 'job_status': 'successful', 'runtime': runtime, 'finput': res_file } Status.make_job_file(dirout, 'generation', name, status)