def update_observation_data(api, sites, observations, start, end): """Post new observation data to all PVDAQ observations from start to end. Parameters ---------- api : solarforecastarbiter.io.api.APISession An active Reference user session. sites: list List of all reference sites as Objects observations: list of solarforecastarbiter.datamodel.Observation List of all reference observations. start : datetime The beginning of the period to request data for. end : datetime The end of the period to request data for. Raises ------ KeyError If NREL_PVDAQ_API_KEY environmental variable is not set. Abuse of KeyError - should probably be ValueError - but kept for consistency with other reference_observations modules. """ nrel_pvdaq_api_key = os.getenv('NREL_PVDAQ_API_KEY') if nrel_pvdaq_api_key is None: raise KeyError('"NREL_PVDAQ_API_KEY" environment variable must be ' 'set to update PVDAQ observation data.') pvdaq_sites = common.filter_by_networks(sites, ['NREL PVDAQ']) for site in pvdaq_sites: common.update_site_observations( api, partial(fetch, nrel_pvdaq_api_key=nrel_pvdaq_api_key), site, observations, start, end)
def test_update_site_observations_gaps(mock_api, mock_fetch, observation_objects_param, fake_ghi_data, mocker): mocker.patch.object(mock_api, 'get_observation_value_gaps', return_value=[ (pd.Timestamp('2019-01-01T12:10Z'), pd.Timestamp('2019-01-01T12:13Z')), (pd.Timestamp('2019-01-01T12:19Z'), pd.Timestamp('2019-01-01T12:20Z')), ]) start = pd.Timestamp('20190101T1205Z') end = pd.Timestamp('20190101T1225Z') site = site_objects[0] common.update_site_observations(mock_api, mock_fetch, site, observation_objects_param, start, end, gaps_only=True) assert mock_api.post_observation_values.call_count == 2 kargs = mock_api.post_observation_values.call_args_list assert kargs[0][0][0] == '' pd.testing.assert_frame_equal( kargs[0][0][1], fake_ghi_data.rename(columns={'ghi': 'value'}) ['2019-01-01T12:10Z':'2019-01-01T12:13Z'].resample( kargs[0][0][1].index.freq).first()) pd.testing.assert_frame_equal( kargs[1][0][1], fake_ghi_data.rename(columns={'ghi': 'value'}) ['2019-01-01T12:19Z':'2019-01-01T12:20Z'].resample( kargs[1][0][1].index.freq).first())
def update_observation_data(api, sites, observations, start, end): """Retrieve data from the network, and then format and post it to each observation at the site. Parameters ---------- api : solarforecastarbiter.io.api.APISession An active Reference user session. sites: list of solarforecastarbiter.datamodel.Site List of all reference sites. observations: list of solarforecastarbiter.datamodel.Observation List of all reference observations. start : datetime The beginning of the period to request data for. end : datetime The end of the period to request data for. Raises ------ KeyError If EIA_API_KEY environmental variable is not set. """ eia_api_key = os.getenv("EIA_API_KEY") if eia_api_key is None: raise KeyError('"EIA_API_KEY" environment variable must be ' 'set to update EIA observation data.') eia_sites = common.filter_by_networks(sites, ['EIA']) for site in eia_sites: common.update_site_observations( api, partial(fetch, eia_api_key=eia_api_key), site, observations, start, end)
def update_observation_data(api, sites, observations, start, end, *, gaps_only=False): """Post new observation data to all reference observations at each SOLRAD site between start and end. api : solarforecastarbiter.io.api.APISession An active Reference user session. sites : list of solarforecastarbiter.datamodel.Site List of all reference sites as Objects observations : list of solarforecastarbiter.datamodel.Observation List of all reference observations. start : datetime The beginning of the period to request data for. end : datetime The end of the period to request data for. gaps_only : bool, default False If True, only update periods between start and end where there are data gaps. """ solrad_sites = common.filter_by_networks(sites, 'NOAA SOLRAD') for site in solrad_sites: common.update_site_observations(api, fetch, site, observations, start, end, gaps_only=gaps_only)
def update_observation_data(api, sites, observations, start, end, *, gaps_only=False): """Post new observation data to a list of DOE RTC Observations from start to end. Parameters ---------- api : solarforecastarbiter.io.api.APISession An active Reference user session. sites: list of solarforecastarbiter.datamodel.Site List of all reference sites as Objects observations: list of solarforecastarbiter.datamodel.Observation List of all reference observations. start : datetime The beginning of the period to request data for. end : datetime The end of the period to request data for. gaps_only : bool, default False If True, only update periods between start and end where there are data gaps. """ doe_rtc_api_key = os.getenv('DOE_RTC_API_KEY') if doe_rtc_api_key is None: raise KeyError('"DOE_RTC_API_KEY" environment variable must be ' 'set to update DOE RTC observation data.') doe_rtc_sites = common.filter_by_networks(sites, 'DOE RTC') for site in doe_rtc_sites: common.update_site_observations( api, partial(fetch, doe_rtc_api_key=doe_rtc_api_key), site, observations, start, end, gaps_only=gaps_only)
def update_observation_data(api, sites, observations, start, end): """Post new observation data to a list of DOE ARM Observations from start to end. api : solarforecastarbiter.io.api.APISession An active Reference user session. sites: list of solarforecastarbiter.datamodel.Site List of all reference sites as Objects observations: list of solarforecastarbiter.datamodel.Observation List of all reference observations. start : datetime The beginning of the period to request data for. end : datetime The end of the period to request data for. """ doe_arm_api_key = os.getenv('DOE_ARM_API_KEY') if doe_arm_api_key is None: raise KeyError('"DOE_ARM_API_KEY" environment variable must be ' 'set to update DOE ARM observation data.') doe_arm_user_id = os.getenv('DOE_ARM_USER_ID') if doe_arm_user_id is None: raise KeyError('"DOE_ARM_USER_ID" environment variable must be ' 'set to update DOE ARM observation data.') doe_arm_sites = common.filter_by_networks(sites, 'DOE ARM') for site in doe_arm_sites: common.update_site_observations( api, partial(fetch, doe_arm_user_id=doe_arm_user_id, doe_arm_api_key=doe_arm_api_key), site, observations, start, end)
def update_observation_data(api, sites, observations, start, end, *, gaps_only=False): """Post new observation data to all MIDC observations from start to end. Parameters ---------- api : solarforecastarbiter.io.api.APISession An active Reference user session. sites: list List of all reference sites as Objects observations: list of solarforecastarbiter.datamodel.Observation List of all reference observations. start : datetime The beginning of the period to request data for. end : datetime The end of the period to request data for. gaps_only : bool, default False If True, only update periods between start and end where there are data gaps. """ midc_sites = common.filter_by_networks(sites, ['NREL MIDC']) for site in midc_sites: common.update_site_observations( api, fetch, site, observations, start, end, gaps_only=gaps_only)
def update_observation_data(api, sites, observations, start, end, *, gaps_only=False): """Post new observation data to a list of SRML Observations from start to end. api : :py:class:`solarforecastarbiter.io.api.APISession` An active Reference user session. sites: list of :py:class:`solarforecastarbiter.datamodel.Site` List of all reference sites as Objects observations: list of :py:class:`solarforecastarbiter.datamodel.Observation` List of all reference observations as Objects start : datetime The beginning of the period to request data for. end : datetime The end of the period to request data for. gaps_only : bool, default False If True, only update periods between start and end where there are data gaps. """ # noqa srml_sites = common.filter_by_networks(sites, 'UO SRML') for site in srml_sites: common.update_site_observations(api, fetch, site, observations, start, end, gaps_only=gaps_only)
def test_update_site_observations_no_data(mock_api, mocker, site_objects_param, observation_objects_param, log, start, end): fetch = mocker.MagicMock() fetch.return_value = pd.DataFrame() common.update_site_observations(mock_api, fetch, site_objects[1], observation_objects_param, start, end) mock_api.assert_not_called()
def test_update_site_observations_uptodate(mock_api, mock_fetch, observation_objects_param, fake_ghi_data, now): site = site_objects[0] mock_api.get_observation_time_range.return_value = (0, now) common.update_site_observations(mock_api, mock_fetch, site, observation_objects_param, None, None) mock_api.post_observation_values.assert_not_called
def test_update_site_observations(mock_api, mock_fetch, site, observation_objects_param, fake_ghi_data): start = pd.Timestamp('20190101T1205Z') end = pd.Timestamp('20190101T1225Z') common.update_site_observations(mock_api, mock_fetch, site, observation_objects_param, start, end) args, _ = mock_api.post_observation_values.call_args assert args[0] == '' pd.testing.assert_frame_equal( args[1], fake_ghi_data.rename(columns={'ghi': 'value'})[start:end])
def test_update_site_observations_no_end(mock_api, mock_fetch, observation_objects_param, fake_ghi_data, now): site = site_objects[0] start = pd.Timestamp('20190101T1210Z') end = None common.update_site_observations(mock_api, mock_fetch, site, observation_objects_param, start, end) args, _ = mock_api.post_observation_values.call_args assert args[0] == '' pd.testing.assert_frame_equal( args[1], fake_ghi_data.rename(columns={'ghi': 'value'})[start:].resample( args[1].index.freq).first())
def test_update_site_observations_out_of_order(mock_api, site_objects_param, mocker, observation_objects_param, fake_ghi_data): start = pd.Timestamp('20190101T1200Z') end = pd.Timestamp('20190101T1230Z') fetch = mocker.MagicMock() fetch.return_value = fake_ghi_data.sample(frac=1) common.update_site_observations(mock_api, fetch, site_objects[1], observation_objects_param, start, end) args, _ = mock_api.post_observation_values.call_args assert args[0] == '' pd.testing.assert_frame_equal( args[1], fake_ghi_data.rename(columns={'ghi': 'value'})[start:end].resample( args[1].index.freq).first())
def test_update_site_observations_no_start(mock_api, mock_fetch, observation_objects_param, fake_ghi_data, now): site = site_objects[0] start = None slimit = pd.Timestamp('20190101T1219Z') end = pd.Timestamp('20190108T1219Z') mock_api.get_observation_time_range.return_value = (0, slimit) common.update_site_observations(mock_api, mock_fetch, site, observation_objects_param, start, end) args, _ = mock_api.post_observation_values.call_args assert args[0] == '' pd.testing.assert_frame_equal( args[1], fake_ghi_data.rename(columns={'ghi': 'value'})[slimit:end].resample( args[1].index.freq).first())
def update_observation_data(api, sites, observations, start, end): """Post new observation data to a list of SRML Observations from start to end. api : solarforecastarbiter.io.api.APISession An active Reference user session. sites: list List of all reference sites as Objects start : datetime The beginning of the period to request data for. end : datetime The end of the period to request data for. """ srml_sites = common.filter_by_networks(sites, 'UO SRML') for site in srml_sites: common.update_site_observations(api, fetch, site, observations, start, end)
def update_observation_data(api, sites, observations, start, end): """Post new observation data to a list of Surfrad Observations from start to end. api : solarforecastarbiter.io.api.APISession An active Reference user session. sites: list of solarforecastarbiter.datamodel.Site List of all reference sites as Objects observations: list of solarforecastarbiter.datamodel.Observation List of all reference observations. start : datetime The beginning of the period to request data for. end : datetime The end of the period to request data for. """ surfrad_sites = common.filter_by_networks(sites, 'NOAA SURFRAD') for site in surfrad_sites: common.update_site_observations(api, fetch, site, observations, start, end)
def update_observation_data(api, sites, observations, start, end): """ Post new observation data to all reference observations at each USCRN site between start and end. api : solarforecastarbiter.io.api.APISession An active Reference user session. sites: list of solarforecastarbiter.datamodel.Site List of all reference sites observations: list of solarforecastarbiter.datamodel.Observation List of all reference observations. start : datetime The beginning of the period to request data for. end : datetime The end of the period to request data for. """ crn_sites = common.filter_by_networks(sites, 'NOAA USCRN') for site in crn_sites: common.update_site_observations(api, fetch, site, observations, start, end)