def upload_signals_to_amphora(client: AmphoraDataRepositoryClient, wz_lc,
amphora_id):
forecasts = load_forecasts(wz_user, wz_password, wz_lc)
# TRANSFORM
signals = []
for f in forecasts:
signals.append(
dict(t=f['utc_time'],
temperature=f['temperature'],
description=f['icon_phrase'],
rainProb=f['rain_prob'],
windSpeed=f['wind_speed'],
windDirection=f['wind_direction'],
cloudCover=f['cloud_cover_percent'],
pressure=f['pressure'],
rainfallRate=f['rate']))
try:
amphora = client.get_amphora(amphora_id)
print(
f'Uploading signals to {amphora.metadata.name} {amphora.metadata.id}'
)
print(f'Properties of first signal val: {signals[0].keys()}')
amphora.push_signals_dict_array(
signals) # this sends the data to Amphora Data
print(f'Sent {len(signals)} signals')
except Exception as e:
print("Exception: %s\n" % e)
raise e
def upload_MLA_signals(client: AmphoraDataRepositoryClient, signals_dict,
amphora_id):
signals = []
success_indicator = 0
try:
for key, value in signals_dict.items():
s = {'t': key, 'price': float(value)}
signals.append(s)
amphora = client.get_amphora(amphora_id)
if len(signals) > 1:
print(
f'Uploading signals to {amphora.metadata.name} {amphora.metadata.id}'
)
print(f'Properties of first signal val: {signals[0].keys()}')
amphora.push_signals_dict_array(
signals) # this sends the data to Amphora Data
print(f'Sent {len(signals)} signals')
else:
print(f'0 signals for {amphora.metadata.name}')
except Exception as e:
print("Exception: %s\n" % e)
success_indicator = 0
raise e
return success_indicator
def create_or_update_amphorae(client: AmphoraDataRepositoryClient, amphora_map,
location_info):
new_map = dict()
try:
# client=amphora_client.ApiClient(configuration)
# amphora_api = amphora_client.AmphoraeApi(client)
for key in amphora_map:
id = amphora_map[key]
if (id == None):
# we have to create an Amphora
wzloc = location_info[key]
locname = wzloc['name']
print(f'Creating new Amphora for location {locname}')
# create the details of the Amphora
name = 'Weather: ' + wzloc['name'] + ' (' + wzloc['state'] + ')'
desc = 'WeatherZone data, from ' + wzloc[
'name'] + '. WeatherZone code: ' + wzloc[
'code'] + ', PostCode: ' + wzloc['postcode']
labels = 'Weather,forecast,timeseries'
ts_cs_id = 'Weatherzone_Forecast'
amphora = client.create_amphora(
name,
desc,
price=2,
labels=labels,
lat=wzloc['latitude'],
lon=wzloc['longitude'],
terms_and_conditions_id=ts_cs_id)
# now create the signals
print("Creating Signals")
for s in signals():
amphora.create_signal(s._property, s.value_type,
s.attributes)
new_map[key] = amphora.id
else:
amphora = client.get_amphora(id)
time.sleep(0.2) # wait a bit to prevent hitting the rate limit
print(f'Using existing amphora: {amphora.metadata.name}')
new_map[key] = id
except Exception as e:
print("Error Create or update amphorae: %s\n" % e)
raise e
return new_map
# Import Amphora librarys
from amphora.client import AmphoraDataRepositoryClient, Credentials
# Import non-Amphora librarys
import os
from datetime import datetime, timedelta
import matplotlib.pyplot as plt
# Login to amphoradata.com
credentials = Credentials(username=os.getenv('username'),
password=os.getenv('password'))
client = AmphoraDataRepositoryClient(credentials)
# Get Amphora of interest
f = open("Amphora_id.txt", "r")
amphora_id = f.read()
# Set time points of interest
start = datetime.utcnow() + timedelta(hours=-48)
end = datetime.utcnow()
dt_range = DateTimeRange(_from=start, to=end)
# Pull data from Amphora
amphora = client.get_amphora(amphora_id)
signals = amphora.get_signals()
df = signals.pull(date_time_range=dt_range).to_pandas(t_as_index=False)
# Make plot of data
df.plot(x='t', y='timeProduct')
plt.show()
# parameters and details
credentials = Credentials(username=os.getenv('username'),
password=os.getenv('password'))
client = AmphoraDataRepositoryClient(credentials)
Electricity_Amphora_id = '89c2e30d-78c8-46ef-b591-140edd84ddb6'
Weather_Amphora_id = '2866d34f-fe2b-4c07-acdd-28fe2d6d9194'
Forecast_amphora_id = '78831a6b-de49-454e-9e5c-7c757478e783'
name = 'South_Australia_Electricity_Price.json'
#############################################################
# Publish raw data on microprediction
mw = MicroWriter(write_key="bdfd44affd28e6c5b45329d6d4df7729")
time_range = a10a.DateTimeRange(_from=datetime.utcnow() + timedelta(hours=-1),
to=datetime.utcnow())
electricity_amphora = client.get_amphora(Electricity_Amphora_id)
electricity_signals = electricity_amphora.get_signals()
df = electricity_signals.pull(date_time_range=time_range).to_pandas()
price = df['price']
mw.set(name=name, value=price[-1])
##########################################################
# Create a prediction and publish on microprediction
# length = mw.num_predictions
# data_time_range = a10a.DateTimeRange(_from = datetime.utcnow() + timedelta(hours=-length/2) , to= datetime.utcnow() )
# pdf = electricity_signals.pull(date_time_range=data_time_range).to_pandas()
# price = pdf['price']
# prices_of_interest = price[-length-1:-1]
# elec_prices_diff = np.zeros(length-1)
# for t in range(1,length):
import os
from datetime import datetime, timedelta
from amphora.client import AmphoraDataRepositoryClient, Credentials
from amphora_api_client import DateTimeRange
# provide your login credentials
credentials = Credentials(username=os.environ['username'],
password=os.environ['password'])
# create a client for interacting with the public Amphora Data Repository
client = AmphoraDataRepositoryClient(credentials)
# get a reference to an Amphora
amphora = client.get_amphora("57d6593f-1889-410a-b1fb-631b6f9c9c85")
# get a reference to the Amphora's signals
signals = amphora.get_signals()
# download some signals and convert to a pandas dataframe
dt_range = DateTimeRange(_from=datetime.utcnow() + timedelta(days=-1),
to=datetime.utcnow() + timedelta(days=2))
df = signals.pull(date_time_range=dt_range,
include_wt=True,
tsi_api="GetEvents").to_pandas()
print(df.describe())
temp = df['temperature']
print(f'Maximum temperature: {temp.max()}')
# update a signal
temperature_signal = signals['temperature']
import os
from datetime import datetime, timedelta
import pytz
import constants
from amphora.client import AmphoraDataRepositoryClient, Credentials
from amphora_api_client import DateTimeRange
from pandas import DataFrame
# login
username = os.getenv('username')
password = os.getenv('password')
credentials = Credentials(username, password)
client = AmphoraDataRepositoryClient(credentials)
insights = client.get_amphora(constants.insights_id)
# save alert method
def alert(name: str, text: str):
# create the file locally
filepath = f'{name}.txt'
file_object = open(filepath, "w+")
file_object.write(text)
file_object.flush()
file_object.close()
insights.push_file(filepath, f'{name}')
print(f'Uploaded file {name}')
def check_max_and_alert(df: DataFrame, key: str, threshold: int,
file_prefix: str, message: str):
import os
from amphora_api_client import Quality
from amphora.client import AmphoraDataRepositoryClient, Credentials
# provide your login credentials
credentials = Credentials(username=os.environ['username'],
password=os.environ['password'])
# create a client for interacting with the public Amphora Data Repository
client = AmphoraDataRepositoryClient(credentials)
# get a reference to an Amphora
amphora = client.get_amphora("e48c9a35-4696-406c-a963-8e72a8bf0131")
quality = Quality() # Quality | The data quality metrics.
quality.accuracy = 2
res = client.amphoraeApi.amphora_quality_set(amphora.amphora_id, quality)
print(res)
# provide your login credentials
credentials = Credentials(username=os.environ['username'], password=os.environ['password'])
# create a client for interacting with the public Amphora Data Repository
client = AmphoraDataRepositoryClient(credentials)
sites = extract.load_wmo_sites()
amphora_map = idcache.load()
max_t = None
# use a random amphora in the list to get the last written time.
# this should be more resilient to failure than always using the first.
if(len(amphora_map) > 0):
random_amphora_id = random.choice(list(amphora_map.values()))
random_amphora = client.get_amphora(random_amphora_id)
max_t = op.get_max_t(random_amphora)
logger.info(f'Max T is {max_t} from Amphora({random_amphora_id})')
for site in sites:
if site.site not in amphora_map:
logger.warn(f'Amphora doesnt exist for site {site.site}')
# then create it
a = op.create_amphora(client, site)
amphora_map[site.site] = a._id
logger.info(f'Amphora({a.amphora_id}) created for site {site.site}')
else:
a = client.get_amphora(amphora_map[site.site])
logger.info(f'Amphora({a.amphora_id}) exists for site {site.site}')
if a is not None:
amphora_ids_file_name = os.getenv('amphora_ids_file_name')
with open(amphora_ids_file_name, 'r') as file:
d_string = file.read()
amphora_ids = json.loads(d_string)
#upload data to amphora data website
credentials = Credentials(username=os.getenv('username'), password=os.getenv('password'))
client = AmphoraDataRepositoryClient(credentials)
#TODO: should update to most recent timestamp rather than all values in the future
#DAILY REPORTS
#upload eyci, estli data, return value in form {'eyci': eyci_signals, ...}
eyci_estli_data = funcs.get_eyci_estli_data()
for report,signals in eyci_estli_data.items():
amphora = client.get_amphora(amphora_ids[report])
success_indicator = push_signal_to_amphora(signals, amphora)
#WEEKLY REPORTS
#upload cattle data, return value in form {data_name: signals, ...}
cattle_data = funcs.get_cattle_data()
for report,signals in cattle_data.items():
amphora = client.get_amphora(amphora_ids[report])
success_indicator = push_signal_to_amphora(signals, amphora)
#upload OTH sheep data
oth_sheep_data = funcs.get_oth_sheep_data()
for report,signals in cattle_data.items():
amphora = client.get_amphora(amphora_ids[report])
success_indicator = push_signal_to_amphora(signals, amphora)
import os
from amphora.client import AmphoraDataRepositoryClient, Credentials
# provide your login credentials
credentials = Credentials(username=os.environ['username'],
password=os.environ['password'],
host="https://app.amphoradata.com")
# create a client for interacting with the public Amphora Data Repository
client = AmphoraDataRepositoryClient(credentials)
# get a reference to an Amphora
amphora = client.get_amphora("84f36667-9bc9-4da1-baba-1ef3160eae0f")
# skip = number of files to ignore
# take= the max num of files in the list
# order_by = the way to order the files. Choose Alphabetical (default) or LastModified
files = amphora.get_files(100, 0)
print(f'The client returned a list with {len(files)} names')
print(f'The first file name is {files[0].name}')
# download a file
for f in files:
print(f'Downloading {f.name}')
f.pull(f'./data/temp/{f.name}')
print('done')
password=os.getenv('password'))
client = AmphoraDataRepositoryClient(credentials)
#try:
# Gets a token
#res = auth_api.authentication_request_token(token_request = token_request)
#configuration.api_key["Authorization"] = "Bearer " + res
# create an instance of the Users API, now with Bearer token
#users_api = amphora_client.UsersApi(amphora_client.ApiClient(configuration))
#me = users_api.users_read_self()
#print(me)
#except ApiException as e:
# print("Exception when calling AuthenticationAPI: %s\n" % e)
amphora_api = a10a.AmphoraeApi(client.apiClient)
amphora = client.get_amphora(id)
Signal = []
try:
if (isup_status):
Signal.append(dict(t=datetime.utcnow(), isup=1))
else:
Signal.append(dict(t=datetime.utcnow(), isup=0))
amphora.push_signals_dict_array(Signal)
except ApiException as e:
print("Exception when calling AmphoraeApi: %s\n" % e)
end = time.time()
mlflow.log_metric("time_to_complete", end - start)
mlflow.log_metric("run_complete", 1)
mlflow.end_run()
