def initialize_site_observations(api, site):
"""Creates an observation at the site for each variable in the PVDAQ
site's file.
Parameters
----------
api : io.api.APISession
API Session object, authenticated for the Reference user.
site : datamodel.Site
The site object for which to create the Observations.
"""
try:
extra_params = common.decode_extra_parameters(site)
except ValueError:
logger.warning('Cannot create reference observations at PVDAQ site '
f'{site.name}, missing required parameters.')
return
site_api_id = int(extra_params['network_api_id'])
with open(DEFAULT_SITEFILE) as fp:
obs_metadata = json.load(fp)['observations']
for obs in obs_metadata:
obs_extra_params = json.loads(obs['extra_parameters'])
if obs_extra_params['network_api_id'] == site_api_id:
obs['site'] = site
observation = Observation.from_dict(obs)
common.check_and_post_observation(api, observation)
def site_to_obs(site):
ep = json.loads(site.extra_parameters)
interval_length = ep['observation_interval_length']
return Observation.from_dict({
'name': 'site ghi',
'variable': 'ghi',
'interval_label': 'ending',
'interval_value_type': 'interval_mean',
'interval_length': interval_length,
'site': site,
'uncertainty': 0,
'extra_parameters': site.extra_parameters
})
def site_obs():
return [
Observation.from_dict({
'name': 'site ghi',
'observation_id': str(uuid.uuid1()),
'variable': 'ghi',
'interval_label': 'ending',
'interval_value_type': 'interval_mean',
'interval_length': '5',
'site': site,
'uncertainty': 0,
'extra_parameters': site.extra_parameters
}) for site in site_objects[:2]
]
def log(mocker):
log = mocker.patch('solarforecastarbiter.io.reference_observations.'
'common.logger')
return log
site_test_observation = Observation.from_dict({
'name':
'site ghi',
'variable':
'ghi',
'interval_label':
'ending',
'interval_value_type':
'interval_mean',
'interval_length':
1,
'site':
site_objects[0],
'uncertainty':
0,
'extra_parameters': ('{"network": "DOE ARM", '
'"observation_interval_length": 1,'
'"network_api_id": "api_id", '
'"network_api_abbreviation": "siteghi"}')
})
invalid_params = {
'name': 'site-invalid-jsonparams',
'latitude': 3,
'longitude': -3,
'elevation': 6,
'timezone': 'Etc/GMT+8',
def create_observation(api, site, variable, extra_params=None, **kwargs):
""" Creates a new Observation for the variable and site. Kwargs can be
provided to overwrite the default arguments to the Observation constructor.
Kwarg options are documented in 'Other Parameters' below but users should
reference the SolarForecastArbiter API for valid Observation field values.
Parameters
----------
api : solarforecastarbiter.io.api.APISession
An APISession with a valid JWT for accessing the Reference Data user.
site : solarforecastarbiter.datamodel.site
A site object.
variable : string
Variable measured in the observation.
extra_params : dict, optional
If provided, this dict will be serialized as the 'extra_parameters'
field of the observation, otherwise the site's field is copied over.
Must contain the keys 'network_api_length', 'network_api_id', and
'observation_interval_length'.
Other Parameters
----------------
name: string
Defaults to `<site.name> <variable>`
interval_label: string
Defaults to 'ending'
interval_value_type: string
Defaults to 'interval_mean'
uncertainty: float or None
Defaults to None.
Returns
-------
created
The datamodel object of the newly created observation.
Raises
------
KeyError
When the extra_parameters, either loaded from the site or provided
by the user is missing 'network_api_abbreviation'
or 'observation_interval_length'
"""
# Copy network api data from the site, and get the observation's
# interval length
if extra_params:
extra_parameters = extra_params
else:
try:
extra_parameters = decode_extra_parameters(site)
except ValueError:
logger.warning(f'Cannot create observations for site {site.name}'
'missing required extra parameters')
return
site_name = site_name_no_network(site)
observation_name = f'{site_name} {variable}'
# Some site names are too long and exceed the API's limits,
# in those cases. Use the abbreviated version.
if len(observation_name) > 64:
site_abbreviation = extra_parameters["network_api_abbreviation"]
observation_name = f'{site_abbreviation} {variable}'
observation = Observation.from_dict({
'name': kwargs.get('name', observation_name),
'interval_label': kwargs.get('interval_label', 'ending'),
'interval_length': extra_parameters['observation_interval_length'],
'interval_value_type': kwargs.get('interval_value_type',
'interval_mean'),
'site': site,
'uncertainty': kwargs.get('uncertainty'),
'variable': variable,
'extra_parameters': json.dumps(extra_parameters)
})
return check_and_post_observation(api, observation)
def initialize_site_observations(api, site):
"""Creates an observation at the site for each variable in
an SRML site's file.
Parameters
----------
api: :py:class:`solarforecastarbiter.io.api.APISession`
site : :py:class:`solarforecastarbiter.datamodel.Site
The site object for which to create Observations.
Notes
-----
Since variables are labelled with an integer instrument
number, Observations are named with their variable and
instrument number found in the source files.
e.g. A SRML file contains two columns labelled, 1001, and
1002. These columns represent GHI at instrument 1 and
instrument 2 respectively. The `pvlib.iotools` package
converts these to 'ghi_1' and 'ghi_2' for us. We use these
labels to differentiate between measurements recorded by
different instruments.
"""
# Request ~month old data at initialization to ensure we get a response.
start = pd.Timestamp.utcnow() - pd.Timedelta('30 days')
end = start
try:
extra_params = common.decode_extra_parameters(site)
except ValueError:
logger.warning('Cannot create reference observations at MIDC site '
f'{site.name}, missing required parameters.')
return
# use site name without network here to build
# a name with the original column label rather than
# the SFA variable
site_name = common.site_name_no_network(site)
try:
site_df = fetch(api, site, start, end)
except error.HTTPError:
logger.error('Could not find data to create observations '
f'for SRML site {site_name}.')
return
else:
if site_df is None:
logger.error('Could not find data to create observations '
f'for SRML site {site_name}.')
return
for variable in srml_variable_map.keys():
matches = [col for col in site_df.columns
if col.startswith(variable)]
for match in matches:
observation_extra_parameters = extra_params.copy()
observation_extra_parameters.update({
'network_data_label': match})
try:
# Here, we pass a name with match instead of variable
# to differentiate between multiple observations of
# the same variable
common.create_observation(
api, site, srml_variable_map[variable],
name=f'{site_name} {match}',
interval_label='beginning',
extra_params=observation_extra_parameters)
except HTTPError as e:
logger.error(
f'Failed to create {variable} observation at Site '
f'{site.name}. Error: {e.response.text}')
with open(DEFAULT_SITEFILE) as fp:
obs_metadata = json.load(fp)['observations']
for obs in obs_metadata:
obs_site_extra_params = json.loads(obs['site']['extra_parameters'])
if obs_site_extra_params['network_api_id'] == extra_params[
'network_api_id']:
obs['site'] = site
observation = Observation.from_dict(obs)
common.check_and_post_observation(api, observation)
def log(mocker):
log = mocker.patch('solarforecastarbiter.io.reference_observations.'
'common.logger')
return log
site_test_observation = Observation.from_dict({
'name':
'site ghi',
'variable':
'ghi',
'interval_label':
'ending',
'interval_value_type':
'interval_mean',
'interval_length':
1,
'site':
site_objects[0],
'uncertainty':
0,
'extra_parameters': ('{"network": "ARM", "observation_interval_length": 1,'
'"network_api_id": "api_id", '
'"network_api_abbreviation": "siteghi"}')
})
invalid_params = {
'name': 'site-invalid-jsonparams',
'latitude': 3,
'longitude': -3,
'elevation': 6,
'timezone': 'Etc/GMT+8',