def upload_signals_to_amphora(irradiation_amphora_id, signals):
#upload data to amphora data website
configuration = Configuration()
auth_api = amphora_client.AuthenticationApi(
amphora_client.ApiClient(configuration))
token_request = amphora_client.TokenRequest(username=os.getenv('username'),
password=os.getenv('password'))
try:
# Gets a token
res = auth_api.authentication_request_token(
token_request=token_request)
configuration.api_key["Authorization"] = "Bearer " + res
# create an instance of the Users API, now with Bearer token
users_api = amphora_client.UsersApi(
amphora_client.ApiClient(configuration))
me = users_api.users_read_self()
print(me)
except ApiException as e:
print("Exception when calling AuthenticationAPI: %s\n" % e)
amphora_api = amphora_client.AmphoraeApi(
amphora_client.ApiClient(configuration))
try:
print(signals)
amphora_api.amphorae_signals_upload_signal_batch(
irradiation_amphora_id, request_body=signals)
except ApiException as e:
print("Exception when calling AmphoraeApi: %s\n" % e)
def upload_signals_to_amphora(site_id, amphora_id, state_service):
if state_service == 'NSW':
var_string = variable_string()
var_dict = variable_codes()
NSW_params = query_params()[state_service]
signals = query_ts_values(site_id, var_string, var_dict, NSW_params)
elif state_service == 'VIC':
var_string = vic_variable_string()
var_dict = vic_variable_codes()
VIC_params = query_params()[state_service]
signals = query_ts_values(site_id, var_string, var_dict, VIC_params)
else:
print(
"error in upload_signals_to_amphora: state_service must be 'NSW' or 'VIC'"
)
return
if signals == None:
print("No signals received")
return
print(signals)
# LOAD
configuration = Configuration()
# Create an instance of the Authentication class
auth_api = amphora_client.AuthenticationApi(
amphora_client.ApiClient(configuration))
token_request = amphora_client.TokenRequest(username=username,
password=password)
try:
print("Logging In")
token = auth_api.authentication_request_token(
token_request=token_request)
configuration.api_key["Authorization"] = "Bearer " + str(token)
print("Logged in")
client = amphora_client.ApiClient(configuration)
amphora_api = amphora_client.AmphoraeApi(client)
amphora = amphora_api.amphorae_read(amphora_id)
print(f'Uploading signals to {amphora.name} {amphora.id}')
print(signals)
amphora_api.amphorae_signals_upload_signal_batch(
amphora.id,
request_body=signals) # this sends the data to Amphora Data
print(f'Sent {len(signals)} signals')
except ApiException as e:
print("Exception: %s\n" % e)
raise e
def establish_connection():
'''
create token and return it
status: working with v0.3.0!
'''
configuration = Configuration()
configuration.host = 'https://beta.amphoradata.com'
# create instance of API class, creates an unauthenticated client
auth_api = amphora_client.AuthenticationApi(
amphora_client.ApiClient(configuration))
# if function throws ValueError 'username'/'password' must not be none, then the environment
# variables are not set, c.f. setup-EForecast.sh and readme.MD section Quickstart
token = auth_api.authentication_request_token(
token_request=amphora_client.TokenRequest(username=useful['username'],
password=useful['password']))
return token
def create_or_update_amphorae(amphora_map, location_info):
# LOAD
configuration = Configuration()
# Create an instance of the Authentication class
auth_api = amphora_client.AuthenticationApi(
amphora_client.ApiClient(configuration))
token_request = amphora_client.TokenRequest(username=os.getenv('username'),
password=os.getenv('password'))
new_map = dict()
try:
print("Logging in")
token = auth_api.authentication_request_token(
token_request=token_request)
configuration.api_key["Authorization"] = "Bearer " + str(token)
print("Logged in")
client = amphora_client.ApiClient(configuration)
amphora_api = amphora_client.AmphoraeApi(client)
#keys are postcodes
for key in amphora_map:
id = amphora_map[key]
if (id == None):
# we have to create an Amphora
ghiloc = location_info[key]
locname = ghiloc['name']
print(f'Creating new Amphora for location {locname}')
# create the details of the Amphora
name = 'Solar Irradiance GHI: ' + ghiloc[
'name'] + ' (' + ghiloc['state'] + ')'
desc = 'Calculated data, from ' + ghiloc['name'] + \
'. PostCode: ' + str(key) + '\r\n' + \
'Properties include: \r\n- temperature (degrees C)\r\n' + \
'- Solar Zenith Angle\r\n- Global Horizontal Irradiance (W/m^2)\r\n' + \
'- Cloud Cover (%)\r\nTemperature and cloud cover signals supplied by Weatherzone'
labels = 'Weather,forecast,solar,timeseries'
ts_cs_id = 'Weatherzone_Forecast'
dto = amphora_client.CreateAmphora(
name=name,
description=desc,
price=0,
lat=ghiloc['lat'],
lon=ghiloc['long'],
labels=labels,
terms_and_conditions_id=ts_cs_id)
res = amphora_api.amphorae_create(create_amphora=dto)
# now create the signals
print("Creating Signals")
for s in signals():
amphora_api.amphorae_signals_create_signal(res.id,
signal=s)
new_map[key] = res.id
else:
a = amphora_api.amphorae_read(id)
print(f'Using existing amphora: {a.name}')
new_map[key] = id
existing_signals = amphora_api.amphorae_signals_get_signals(id)
if (len(existing_signals) > 0):
print('Signals exist already')
else:
print('Adding signals')
for s in signals():
amphora_api.amphorae_signals_create_signal(id,
signal=s)
except ApiException as e:
print("Error Create or update amphorae: %s\n" % e)
raise e
return new_map
load_dotenv()
import amphora_client
from amphora_client.rest import ApiException
from amphora_client.configuration import Configuration
import os
from src.MLA import *
EYCI_id = "1bd74490-ccf6-43d1-99f1-ef53f60c293c"
ESTLI_id = "35f32a02-5abf-417b-a8ec-40dc7ac3ff7d"
EYCI_dict, ESTLI_dict = get_EYCI_ESTLI_data()
#upload data to amphora data website
configuration = Configuration()
auth_api = amphora_client.AuthenticationApi(
amphora_client.ApiClient(configuration))
token_request = amphora_client.TokenRequest(username=os.getenv('username'),
password=os.getenv('password'))
try:
# Gets a token
res = auth_api.authentication_request_token(token_request=token_request)
configuration.api_key["Authorization"] = "Bearer " + res
# create an instance of the Users API, now with Bearer token
users_api = amphora_client.UsersApi(
amphora_client.ApiClient(configuration))
me = users_api.users_read_self()
except ApiException as e:
print("Exception when calling AuthenticationAPI: %s\n" % e)
amphora_api = amphora_client.AmphoraeApi(
def create_or_update_amphorae(amphora_map, location_info):
# LOAD
configuration = Configuration()
# Create an instance of the Authentication class
auth_api = amphora_client.AuthenticationApi(
amphora_client.ApiClient(configuration))
token_request = amphora_client.TokenRequest(username=username,
password=password)
new_map = dict()
try:
print("Logging in")
token = auth_api.authentication_request_token(
token_request=token_request)
configuration.api_key["Authorization"] = "Bearer " + str(token)
print("Logged in")
client = amphora_client.ApiClient(configuration)
amphora_api = amphora_client.AmphoraeApi(client)
for key in amphora_map:
id = amphora_map[key]
if (id == None):
# we have to create an Amphora
waterloc = location_info[key]
locname = waterloc['name']
print(f'Creating new Amphora for location {locname}')
# create the details of the Amphora
name = 'Water Information: ' + waterloc[
'name'] + ' (' + waterloc['state'] + ')'
desc = 'WaterNSW data, from ' + waterloc[
'name'] + '. WaterNSW site id: ' + key
labels = 'Water,actuals,timeseries'
ts_cs_id = 'Creative_Commons_4p0'
#TODO: add terms_and_conditions_id when it's on the website
#terms_and_conditions_id = ''
dto = amphora_client.CreateAmphora(
name=name,
description=desc,
labels=labels,
price=0,
lat=waterloc['lat'],
lon=waterloc['long'],
terms_and_conditions_id=ts_cs_id)
res = amphora_api.amphorae_create(create_amphora=dto)
# now create the signals
print("Creating Signals")
for s in signals():
amphora_api.amphorae_signals_create_signal(res.id,
signal=s)
new_map[key] = res.id
else:
a = amphora_api.amphorae_read(id)
print(f'Using existing amphora: {a.name}')
new_map[key] = id
existing_signals = amphora_api.amphorae_signals_get_signals(id)
if (len(existing_signals) > 0):
print('Signals exist already')
else:
print('Adding signals')
for s in signals():
amphora_api.amphorae_signals_create_signal(id,
signal=s)
except ApiException as e:
print("Error Create or update amphorae: %s\n" % e)
raise e
return new_map
# Import Amphora librarys
import amphora_client as a10a
from amphora_client.configuration import Configuration
from amphora_extensions.file_uploader import FileUploader
# Import non-Amphora librarys
from array import array
import os
import numpy as np
import time
from datetime import datetime, timedelta
# Login to amphoradata.com
configuration = Configuration()
configuration.host = "https://app.amphoradata.com"
auth_api = a10a.AuthenticationApi(a10a.ApiClient(configuration))
token_request = a10a.TokenRequest(username=os.getenv('username'),
password=os.getenv('password'))
res = auth_api.authentication_request_token(token_request=token_request)
configuration.api_key["Authorization"] = "Bearer " + res
amphora_api = a10a.AmphoraeApi(a10a.ApiClient(configuration))
# Define model function
def time_product(date_time):
time_hour = date_time.hour
time_minute = date_time.minute
time_second = date_time.second
time_prod = time_hour * time_minute * time_second
def get_temps_cloud(id, start_date, end_date, required_data):
configuration = Configuration()
configuration.host = "https://beta.amphoradata.com"
# Create an instance of the auth API class
auth_api = a10a.AuthenticationApi(a10a.ApiClient(configuration))
token_request = a10a.TokenRequest(username=os.getenv('username'),
password=os.getenv('password'))
try:
# Gets a token
t1_start = time.perf_counter()
res = auth_api.authentication_request_token(
token_request=token_request)
t1_stop = time.perf_counter()
print("Elapsed time:",
t1_stop - t1_start) # print performance indicator
configuration.api_key["Authorization"] = "Bearer " + res
amphora_api = a10a.AmphoraeApi(a10a.ApiClient(configuration))
print(f'Getting signals for: {amphora_api.amphorae_read(id).name}')
signals = amphora_api.amphorae_signals_get_signals(id)
properties = list((s._property for s in signals))
print(properties)
ts_api = a10a.TimeSeriesApi(a10a.ApiClient(
configuration)) # the API for interacting with time series
# Create a DateTimeRange to describe over what period we want data.
time_range = a10a.DateTimeRange(_from=start_date, to=end_date)
# Create a variable object for getting temperature data
temperatureVariable = a10a.NumericVariable(
kind="numeric",
value=a10a.Tsx(tsx="$event.temperature"),
aggregation=a10a.Tsx("avg($value)"))
get_series = a10a.GetSeries(
[id],
search_span=time_range,
inline_variables={"temperature": temperatureVariable})
time_series_data = ts_api.time_series_query_time_series(
a10a.QueryRequest(get_series=get_series))
print(
f'Got {len(time_series_data.timestamps)} datapoints and {len(time_series_data.properties)} properties'
)
# access the data in time_series_data.properties
print("-----------")
temp_values = next(value for value in time_series_data.properties
if value.name == 'temperature')
# Create a variable object for getting cloudCover data
cloudCoverVariable = a10a.NumericVariable(
kind="numeric",
value=a10a.Tsx(tsx="$event.cloudCover"),
aggregation=a10a.Tsx("avg($value)"))
get_series = a10a.GetSeries(
[id],
search_span=time_range,
inline_variables={"cloudCover": cloudCoverVariable})
time_series_data = ts_api.time_series_query_time_series(
a10a.QueryRequest(get_series=get_series))
print(
f'Got {len(time_series_data.timestamps)} datapoints and {len(time_series_data.properties)} properties'
)
# access the data in time_series_data.properties
print("-----------")
cloudCover_values = next(value for value in time_series_data.properties
if value.name == 'cloudCover')
wtVar = a10a.NumericVariable(kind="numeric",
value=a10a.Tsx(tsx="$event.wt"),
aggregation=a10a.Tsx("avg($value)"))
get_series = a10a.GetSeries([id],
search_span=time_range,
inline_variables={"wt": wtVar})
time_series_data = ts_api.time_series_query_time_series(
a10a.QueryRequest(get_series=get_series))
wt_values = next(value for value in time_series_data.properties
if value.name == 'wt')
#create dictionary of signals, with timestamp: [temp, cloud_cover] for each entry if both required
temp_cloud_dict = {}
wt_dict = {}
for i in range(len(temp_values.values)):
#update the dictionary with the value for the most recent write_time
#and override wt_dict entries if so
if time_series_data.timestamps[i] not in temp_cloud_dict.keys() or \
wt_values.values[i] > wt_dict[time_series_data.timestamps[i]]:
if required_data == 'both':
temp_cloud_dict[time_series_data.timestamps[i]] = [
temp_values.values[i], cloudCover_values.values[i]
]
elif required_data == 'cloud_cover':
temp_cloud_dict[time_series_data.timestamps[
i]] = cloudCover_values.values[i]
elif required_data == 'temperature':
temp_cloud_dict[
time_series_data.timestamps[i]] = temp_values.values[i]
else:
raise ValueError(
"required_data must be 'both', 'temperature' or 'cloud_cover'"
)
wt_dict[time_series_data.timestamps[i]] = wt_values.values[i]
except ApiException as e:
print("Exception when calling API: %s\n" % e)
return temp_cloud_dict
def fetch_data(params):
'''
Fetches dataset in amphora ids, returns a pd.DataFrame of the API response
params: parameter dict containing keys
id e.g. ecc5263e-83b6-42d6-8852-64beffdf204e
token as created by establish_connection()
start_time datetime object
end_time datetime object
filter optional parameter to filter results by
returns
pd.DataFrame
'''
id = params['id']
start_time = params['start_time']
end_time = params['end_time']
if 'filter' in params.keys():
filter_ = params['filter']
else:
filter_ = False
# example ids to test run this function with:
# ids= {'electricity_NSW': 'ecc5263e-83b6-42d6-8852-64beffdf204e',
# 'weather_QLD': 'a46f461f-f7ee-4cc5-a1e4-569960ea5ed8',
# 'electricity_SA': '89c2e30d-78c8-46ef-b591-140edd84ddb6',
# 'electricity_QLD': 'ef22fa0f-010c-4ab1-8a28-a8963f838ce9',
# 'electricity_VIC': '3b66da5a-0723-4778-98fc-02d619c70664',
# 'weather_NSW': '11fd3d6a-12e4-4767-9d52-03271b543c66',
# 'weather_VIC': 'd48ac35f-c658-41c1-909a-f662d6f3a972',
# 'weather_SA': 'f860ba45-9dda-41e0-91aa-73901a323318'}
configuration = Configuration()
configuration.host = "https://beta.amphoradata.com"
configuration.api_key["Authorization"] = "Bearer " + establish_connection()
# To see amphora belonging to the id on the web:
# print("https://beta.amphoradata.com/Amphorae/Detail?id={}".format(id))
try:
amphora_api = amphora_client.AmphoraeApi(
amphora_client.ApiClient(configuration))
#print(f'Getting signals for: {amphora_api.amphorae_read(id).name}')
signals = amphora_api.amphorae_get_signals(id)
# grab all the different timeseries names using a listcomprehension
properties = [s._property for s in signals]
# the API for interacting with time series
ts_api = amphora_client.TimeSeriesApi(
amphora_client.ApiClient(configuration))
time_range = amphora_client.DateTimeRange(_from=start_time,
to=end_time)
# get tomorrow's temperatures
property_ = []
value_dict = {}
if filter_:
for _ in properties:
property_ = '$event.' + _
variable1 = amphora_client.NumericVariable(
kind="numeric",
value=amphora_client.Tsx(tsx=property_),
filter=amphora_client.Tsx(
"$event.periodType.String = '{}'".format(filter_)),
aggregation=amphora_client.Tsx("avg($value)"))
value_dict[_] = variable1
else:
for _ in properties:
property_ = '$event.' + _
variable1 = amphora_client.NumericVariable(
kind="numeric",
value=amphora_client.Tsx(tsx=property_),
aggregation=amphora_client.Tsx("avg($value)"))
value_dict[_] = variable1
# if some of the data needs to be filtered, e.g., ecc5263e-83b6-42d6-8852-64beffdf204e
# has 'Forecast' and 'Actual' values for spot prices, also get the second one if
# the parameter is passed
# if you dont understand wtf is going on:
# https://docs.microsoft.com/en-au/rest/api/time-series-insights/dataaccess(preview)/query/execute#getseries
# get the series (plural, if so defined above in value_dict!), save to time_series_data
get_series = amphora_client.GetSeries([id],
search_span=time_range,
inline_variables=value_dict)
time_series_data = ts_api.time_series_query_time_series(
amphora_client.QueryRequest(get_series=get_series))
# ingest data into pd.DataFrame
# create list of returned timeseries
apple = []
for _ in range(len(time_series_data.properties)):
apple.append(
np.array(time_series_data.properties[_].values).reshape(
(len(time_series_data.properties[_].values))))
# dataframe containing all data for further stuff
banana = pd.DataFrame(apple).T
banana.index = np.array(time_series_data.timestamps)
# delete weird artefact
del banana[0]
if filter_:
properties_filtered = [p + '.' + filter_ for p in properties]
banana.columns = properties_filtered
else:
banana.columns = properties
return banana
except ApiException as e:
print("Exception when calling API: %s\n" % e)
def upload_series(df, params1, id_=''):
'''
Fetches dataset in amphora ids, returns a pd.DataFrame of the API response
df: pandas DataFrame
id_: (str) if amphora exists pass id, else pass empty string
params: parameter dict containing keys
name (str) name of amphora e.g. 'Price Predictions Avocado Futures'
description (str) description of amphora e.g. 'Spot Price Predictions for Futures of
Australian grown Hass Avocadoes.'
price (int) price of Amphora, can be >=0
file_path (str) path to files, or filename of file to be uploaded.
returns:
failure/success message
'''
name = params1['name']
description = params1['description']
price = params1['price']
# file_path = params1['file_path']
configuration = Configuration()
configuration.host = "https://beta.amphoradata.com"
configuration.api_key["Authorization"] = "Bearer " + establish_connection()
try:
# get API token and create authorized client:
configuration.api_key[
"Authorization"] = "Bearer " + establish_connection()
amphora_api = amphora_client.AmphoraeApi(
amphora_client.ApiClient(configuration))
# create amphora description (dto), then create amphora (amphora)
if not id_:
dto = amphora_client.CreateAmphoraDto(name=name,
price=price,
description=description)
amphora = amphora_api.amphorae_create(create_amphora_dto=dto)
signals_liste = []
for col in df.columns:
signals_liste.append(
SignalDto(_property=col, value_type='Numeric'))
for _ in signals_liste:
amphora_api.amphorae_create_signal(amphora.id, signal_dto=_)
else:
amphora = amphora_api.amphorae_read(id_)
signal = _contained_signals(amphora_api, id_)[0]
# test that all signals are exist both in the amphora and the df:
cols = [n for n in df.columns]
assert all(
elem in signal for elem in cols
), f'Content, i.e. signals, in Amphora do not match with DataFrame columns. Amphora contains: {signal}, df contains {cols}'
# transform pd.DataFrame into API compatible format of a list of dicts (1 dict per df row)
signal_list = []
cats = [x for x in df.columns]
for row in range(df.shape[0]):
temp_dict = dict(t=df.index[row])
for _ in range(len(cats)):
temp_dict[cats[_]] = float(df.iloc[row, _])
signal_list.append(temp_dict)
# upload of signals
amphora_api.amphorae_upload_signal_batch(amphora.id,
request_body=signal_list)
print(
f'Uploaded {len(signal_list)} signals to amphora "{amphora.name}" under "{amphora.id}"'
)
# if one where to upload a file:
# uploader = FileUploader(amphora_api)
#
# the way it is set ip, file_path can either be an entire directory
# corresponding to head in (head, tail = ntpath.split(path)),
# or to tail, the filename, if there is a valid filename!
# uploader.create_and_upload_file(amphora.id, file_path)
except ApiException as e:
print("Exception when calling API: %s\n" % e)
