def test_cancel_live():
write_key = new_key(difficulty=7)
predictor = MicroWriter(write_key=write_key)
values = sorted(np.random.randn(predictor.num_predictions))
predictor.submit(name='cop.json', values=values, delay=predictor.DELAYS[0])
res = predictor.cancel(name='cop.json')
assert res, "Cancellation of submission failed"
def test_is_valid():
key = "4c3db49d3a291acad7bdedecfa787891"
assert MicroWriter.is_valid_key(key)
key = b"4c3db49d3a291acad7bdedecfa787891"
assert MicroWriter.is_valid_key(key) == False, "Binary keys are not valid"
key = "3c3db49d3a291acad7bdedecfa787891"
assert not MicroWriter.is_valid_key(key)
def run():
try:
mw = MicroWriter(write_key=TRAFFIC_WRITE_KEY)
except:
raise Exception(
"You need to set the write key for this example to work")
while True:
value = hospital_bike_activity(station_ids)
utc_now = pytz.utc.localize(datetime.utcnow())
pst_now = utc_now.astimezone(pytz.timezone("America/Los_Angeles"))
res = mw.set(name=NAME, value=float(value))
pprint({
'PST time': pst_now.strftime("%H:%M"),
'activity': value,
"res": res
})
print('', flush=True)
def test_repo_live():
if DODDLE_MAMMAL is not None:
mw = MicroWriter(write_key=DODDLE_MAMMAL,
base_url='https://devapi.microprediction.org')
saved_url = mw.get_own_repository()
res = mw.set_repository(
url=
'https://www.linkedin.com/in/petercotton/detail/recent-activity/posts/'
)
print(mw.get_own_repository())
print(res)
res = mw.delete_repository()
mw.set_repository(url=saved_url)
saved_back = mw.get_own_repository()
assert saved_back == saved_url
def example_z2_prediction(write_key, name=None):
"""
:param write_key str See above for creating a write_key
:param name str Name of a bivariate stream, such as 'z2~helicopter_psi~helicopter_theta~70.json'
"""
if name is None:
name = 'z2~helicopter_psi~helicopter_theta~70.json'
copula = fit_copula_to_z2_data()
submitted_points = copula.sample(mr.num_predictions)
qpitch = submitted_points[0]
qyaw = submitted_points[1]
values = sorted([project(p, q) for p, q in zip(qpitch, qyaw)])
# Now submit them
mw = MicroWriter(write_key=write_key)
print(mw.animal + ' is submitting ')
horizon = 70
reply = mw.submit(name=name, values=values, delay=horizon)
pprint(reply)
def run():
try:
mw = MicroWriter(write_key=TRAFFIC_WRITE_KEY)
except:
raise Exception(
"You need to set the write key for this example to work")
while True:
value = number_of_articles_that_mention_cuomo_or_nyc()
utc_now = pytz.utc.localize(datetime.utcnow())
pst_now = utc_now.astimezone(pytz.timezone("America/Los_Angeles"))
try:
res = mw.set(name=NAME, value=float(value))
pprint({
'PST time': pst_now.strftime("%H:%M"),
'count': value,
"res": res
})
print('', flush=True)
except:
continue
#
# Work in progress ... broken currently
#
# Example of sending live data to www.microprediction.com to predict a simulated sequence
# We generate a time series of the number of people who are infected in a model for an epidemic
# that is similar to CIR but takes novelty of interaction into account. see https://www.overleaf.com/read/sgjvfxydcwpk
# The time series comprises a sequence of daily observations (real time) updated every 90 seconds.
NAMES = ['susceptible', 'infected', 'recovered']
DT = 90 # Seconds between each update
try:
from microprediction.config_private import MATCHABLE_BAT
mw = MicroWriter(write_key=MATCHABLE_BAT)
except:
raise Exception(
"You need to set the write key for this example to work, and it must be 'copula strength' "
)
def entire(a, t):
if a * t > 1e-4:
return (1 - math.exp(-a * t)) / (a * t)
else:
return 1.0
def compartmental(t, y, a, b, gamm):
"""
def test_new():
write_key = new_key(difficulty=7)
assert MicroWriter.is_valid_key(write_key)
##############################################################
# parameters and details
credentials = Credentials(username=os.getenv('username'),
password=os.getenv('password'))
client = AmphoraDataRepositoryClient(credentials)
Electricity_Amphora_id = '89c2e30d-78c8-46ef-b591-140edd84ddb6'
Weather_Amphora_id = '2866d34f-fe2b-4c07-acdd-28fe2d6d9194'
Forecast_amphora_id = '78831a6b-de49-454e-9e5c-7c757478e783'
name = 'South_Australia_Electricity_Price.json'
#############################################################
# Publish raw data on microprediction
mw = MicroWriter(write_key="bdfd44affd28e6c5b45329d6d4df7729")
time_range = a10a.DateTimeRange(_from=datetime.utcnow() + timedelta(hours=-1),
to=datetime.utcnow())
electricity_amphora = client.get_amphora(Electricity_Amphora_id)
electricity_signals = electricity_amphora.get_signals()
df = electricity_signals.pull(date_time_range=time_range).to_pandas()
price = df['price']
mw.set(name=name, value=price[-1])
##########################################################
# Create a prediction and publish on microprediction
# length = mw.num_predictions
# data_time_range = a10a.DateTimeRange(_from = datetime.utcnow() + timedelta(hours=-length/2) , to= datetime.utcnow() )
# pdf = electricity_signals.pull(date_time_range=data_time_range).to_pandas()
# price = pdf['price']
# prices_of_interest = price[-length-1:-1]
# get started creating streams (see the 4th module in particular)
import logging
import urllib
import time
import pytz
import random
from datetime import datetime
from pprint import pprint
from microprediction import MicroWriter
import os
write_key = os.environ.get(
'WRITE_KEY'
) # GitHub action needs to set env variable. You need to create a GitHub secret called WRITE_KEY
mw = MicroWriter(write_key=write_key)
assert mw.key_difficulty(
mw.write_key) >= 13, "You need a key of difficulty 13 for copula streams"
mw.set_repository(
url=
'https://github.com/microprediction/microprediction/blob/master/microprediction/live/seattle_wind.py'
) # courtesy
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
def wait_between_attempts():
""" Incremental backoff between connection attempts """
wait_time = 5.3 # time is in seconds
while True:
#!/usr/bin/python3.8
from microprediction import MicroWriter
from apscheduler.schedulers.blocking import BlockingScheduler
import numpy as np
from pprint import pprint
from scipy.integrate import odeint
from math import sqrt
# Example of sending live data to www.microprediction.org and kick-starting copula streams
# This uses the MicroWriter directly, though soon there will be a parent class that makes this more convenient
NAMES = ['three_body_x.json', 'three_body_y.json', 'three_body_z.json']
try:
from microprediction.config_private import THREE_BODY_WRITE_KEY
mw = MicroWriter(write_key=THREE_BODY_WRITE_KEY)
except:
raise Exception(
"You need to set the write key for this example to work, and it must be 'copula strength' "
)
SCALE = 10**8
Y0 = [384.6 * 10**6, 0, 0, 1197000, -9928000, 0, 0, 1025, 0, 8490, -2455, 100]
print("Initial value is " + str([y / SCALE for y in Y0[:3]]), flush=True)
Y = Y0
def threebody(Y, t):
"""
Forked from https://guillaumecantin.pythonanywhere.com/animation/4/
"""
m1 = 5.974 * 10**24 # Earth
from microprediction import MicroWriter
import time
from pprint import pprint
import numpy as np
# This example illustrates how easy it is to enter z1~ streams
# For a video explanation of z1 streams and see https://www.linkedin.com/feed/update/urn:li:activity:6696569137365430272/
# See https://www.linkedin.com/pulse/short-introduction-z-streams-peter-cotton-phd/ for an explanation of z-streams.
# You need to set a write key.
# Supplying a URL to your repository is optional.
# This will do is make your position on the leaderboards clickable and direct people to your work.
# You can alternatively provide your personal homepage or LinkedIn profile
WRITE_KEY = DODDLE_MAMMAL
ANIMAL = MicroWriter.animal_from_key(DODDLE_MAMMAL)
REPO = 'https://github.com/microprediction/microprediction/blob/master/crawler_alternatives/' + ANIMAL.lower(
).replace(' ', '_') + '.py'
# We simply loop over all z1~ streams
# These streams are approximately normally distributed.
# We enter N(0,1) distributed samples so as to try to take advantage of algorithms
# which accidentally submit distributional predictions far from this.
# This code could be run as a cron job or scheduled task at PythonAnywhere, for example
def jiggle():
return 0.001 * np.random.randn()
if __name__ == "__main__":
from microprediction import new_key, MicroWriter
write_key = new_key(difficulty=9)
mw = MicroWriter(write_key=write_key)
die_rolls = [k - 2.5 for k in range(6)] * 50
values = die_rolls[:mw.num_predictions]
res = mw.submit(name='die.json', values=values)
print('https://api.microprediction.org/home/' + write_key)
#!/usr/bin/python3.8
from microprediction import MicroWriter
from microprediction.live import bronx_speed
from apscheduler.schedulers.blocking import BlockingScheduler
from pprint import pprint
import datetime
# Example of sending live data change to www.microprediction.com
try:
from microprediction.config_private import TRAFFIC_WRITE_KEY
mw = MicroWriter(write_key=TRAFFIC_WRITE_KEY)
except:
raise Exception("You need to set the write key for this example to work")
NAME = "bronx_traffic_speed_on_change.json"
prev_speed = None
def poll_and_send_if_changed():
""" Modify feed of speed changes - or at least keep it warm """
global speed, prev_speed
if prev_speed is None:
prev_speed = bronx_speed()
speed = None
else:
speed = bronx_speed()
try:
fspeed = float(speed)
fprev = float(prev_speed)
def test_client():
with Timer() as t:
write_key = new_key(difficulty=7)
mw = MicroWriter(write_key=write_key)
assert (t.elapsed < 15)
from microprediction import MicroWriter
import os
import random
SECONDS = 60*15 # How long to spend making a throwaway key
if __name__=='__main__':
# WRITE_KEY is injected into environ by GitHub action. Grab it.
write_key = os.environ.get('WRITE_KEY')
if ',' in write_key:
write_keys = write_key.split(',')
write_key = random.choice(write_keys)
# Create a throwaway key and use it to bolster the balance, using a transfer.
if write_key is not None:
mw = MicroWriter(write_key=write_key)
balance = float(mw.get_balance())
if balance<0:
print('Trying to top up balance for '+mw.animal, ' which is currently '+str(balance))
throwaway_key = mw.maybe_create_key(difficulty=11, seconds=SECONDS)
if throwaway_key is None:
print('Sorry, no luck this time')
else:
print('Hit the jackpot ... topping up balance for '+ mw.animal)
mw.put_balance(source_write_key=throwaway_key, amount=256)
print('New balance is '+str(mw.get_balance()))
else:
print('Balance for '+mw.animal+' is '+str(balance)+' so no need to top up.')
# 1. Grab the Github secrets
try:
# For this script to work you need to create four separate GitHub secrets
# called WRITE_KEY_1 WRITE_KEY_2 WRITE_KEY_3 and WRITE_KEY_4
# The idea is that this way you get 900 samples instead of 225
WRITE_KEYS = [os.environ.get('WRITE_KEY_' + str(i + 1)) for i in range(4)]
assert len(WRITE_KEYS) == 4, 'Need four write keys to make the syndicate'
except:
# Or one secret called WRITE_KEY or WRITE_KEYS with them comma separated
WRITE_KEYS_comma_sep = os.environ.get('WRITE_KEYS') or os.environ.get(
'WRITE_KEY')
WRITE_KEYS = WRITE_KEYS_comma_sep.split(',')
print('Copula syndicate is firing up.')
for write_key in WRITE_KEYS:
animal = MicroWriter.animal_from_key(write_key)
print(animal)
# 2. Pick a copula
VINES = [
'center', 'regular', 'direct'
] # See https://sdv.dev/Copulas/tutorials/03_Multivariate_Distributions.html#Vine-Copulas
VINE_TYPE = random.choice(
VINES
) # Perhaps you want to fix this choice. This way we get lots of plots.
# 3. (Optional) Set the URL of your repo so that others can learn from it
REPO = 'https://github.com/microprediction/microactors-plots/blob/master/fit.py' # <--- Change your username
PLOTS_PATH = os.path.dirname(
os.path.realpath(__file__)) + os.path.sep + 'gallery'
def test_delays_2():
mc = MicroWriter(write_key=create_key(difficulty=7))
assert len(mc.DELAYS) == 3
import os
from getjson import getjson
from microprediction import MicroWriter
# This code lies at https://github.com/microprediction/chess/blob/main/set.py
# and generates streams such as https://www.microprediction.org/stream_dashboard.html?stream=chess_bullet_level_Hikaru
write_key = os.environ.get('WRITE_KEY') # GitHub action needs to set env variable. You need to create a GitHub secret called WRITE_KEY
mw = MicroWriter(write_key=write_key)
assert mw.key_difficulty(mw.write_key)>=12, "You need a key of difficulty 12 to create a stream"
PLAYERS = {'Hikaru': 'hikaru_nakamura',
'Firouzja2003': 'alireza_firouzja',
'GMWSO': 'wesley_so',
'LyonBeast': 'maxime_vachier_lagrave',
'nihalsarin': 'nihal_sarin',
'DanielNaroditsky': 'daniel_naroditsky',
'PinIsMightier': 'halloween_gambit'}
URL_TEMPLATE = 'https://api.chess.com/pub/player/HANDLE/stats'
CATEGORIES = ['chess_blitz', 'chess_bullet']
if __name__ == '__main__':
for category in CATEGORIES:
for handle, player in PLAYERS.items():
url = URL_TEMPLATE.replace('HANDLE', handle.lower())
data = getjson(url)
if data is not None:
current_value = int(data[category]['last']['rating'])
level_name = category + '_level_' + handle + '.json' # Name of stream with rating level
from microprediction import new_key, MicroWriter
from pprint import pprint
write_key = new_key(difficulty=9)
mw = MicroWriter(write_key=write_key)
# New video tutorials are available at https://www.microprediction.com/python-1 to help you
# get started running crawlers at www.microprediction.com
if __name__ == '__main__':
die_rolls = [k - 2.5 for k in range(6)] * 50
values = die_rolls[:mw.num_predictions]
pprint(mw.submit(name='die.json', values=values))
print('https://api.microprediction.org/home/' + write_key)
import os
from getjson import getjson
from microprediction import MicroWriter
write_key = os.environ.get(
'WRITE_KEY') # GitHub action needs to set env variable. You need to create a GitHub secret called WRITE_KEY
mw = MicroWriter(write_key=write_key)
assert mw.key_difficulty(mw.write_key) >= 12, "You need a key of difficulty 12 to create a stream"
CATEGORIES = ['stargazers_count', 'forks_count']
REPOS = {'pykalman': 'https://github.com/pykalman/pykalman',
'microprediction': 'https://github.com/microprediction/microprediction',
'prophet': 'https://github.com/facebook/prophet',
'neuralprophet': 'https://github.com/ourownstory/neural_prophet',
'river': 'https://github.com/online-ml/river',
'scikit-learn': 'https://github.com/scikit-learn/scikit-learn',
'tensorflow': 'https://github.com/tensorflow/tensorflow',
'keras': 'https://github.com/keras-team/keras'}
REPO_APIS = dict( [ (k, v.replace('https://github.com/', 'https://api.github.com/repos/')) for k, v in REPOS.items() ] )
if __name__ == '__main__':
for repo_name, repo_url in REPO_APIS.items():
data = getjson(repo_url)
if data is not None:
for category in CATEGORIES:
current_value = int(data[category])
level_name = 'github_' + category + '_' + repo_name + '.json'
print(mw.set(name=level_name, value=current_value))
else:
print('Something is wrong')
import numpy as np
from pprint import pprint
import matplotlib.pyplot as plt
import random
import time
import warnings
warnings.filterwarnings('ignore')
from copulas.multivariate import GaussianMultivariate
import pandas as pd
# Grab the Github secret
import os
Doodle_Fox = os.environ.get('Doodle_Fox')
#
WRITE_KEY = Doodle_Fox
ANIMAL = MicroWriter.animal_from_key(WRITE_KEY)
REPO = 'https://github.com/spikeshr/microactors/blob/master/fit.py'
print('This is ' + ANIMAL)
VERBOSE = False
# Get historical data, fit a copula, and submit
def fit_and_sample(lagged_zvalues: [[float]], num: int, copula=None):
""" Example of fitting a copula function, and sampling
lagged_zvalues: [ [z1,z2,z3] ] distributed N(0,1) margins, roughly
copula : Something from https://pypi.org/project/copulas/
returns: [ [z1, z2, z3] ] representative sample
"""
# Remark: It's lazy to just sample synthetic data
from microprediction import MicroWriter
from microprediction.samplers import exponential_bootstrap
import time
from pprint import pprint
from microprediction.config_private import OXEATE_MAMMAL
# Enters all the z2 streams
WRITE_KEY = OXEATE_MAMMAL
ANIMAL = MicroWriter.animal_from_key(WRITE_KEY)
print(ANIMAL, flush=True)
STOP_LOSS = 250.
if __name__ == "__main__":
mw = MicroWriter(write_key=WRITE_KEY)
LONG = '~' + str(mw.DELAYS[-1])
SHORT = '~' + str(mw.DELAYS[0])
# Give up if things are going badly ... do it nicely
for _ in range(5):
mw.cancel_worst_active(stop_loss=STOP_LOSS, num=1)
time.sleep(1)
for name in mw.get_streams():
if 'z2~' in name and (LONG in name or SHORT in name):
num = mw.num_predictions
z11 = '~'.join(['z1', name.split('~')[1], name.split('~')[3]])
z12 = '~'.join(['z1', name.split('~')[2], name.split('~')[3]])
lagged1 = mw.get_lagged_values(z11)
lagged2 = mw.get_lagged_values(z12)
if len(lagged1) > 10:
from microprediction import MicroWriter
import numpy as np
from pprint import pprint
import matplotlib.pyplot as plt
import random
import time
import warnings
warnings.filterwarnings('ignore')
from copulas.multivariate import GaussianMultivariate
import pandas as pd
# Grab the Github secret
import os
WRITE_KEY = os.environ.get('WRITE_KEY') # <-- You need to add a Github secret
ANIMAL = MicroWriter.animal_from_key(WRITE_KEY) # <-- Your nom de plume
REPO = 'https://github.com/microprediction/microactors/blob/master/fit.py' # <--- Change your username
print('This is ' + ANIMAL + ' firing up')
STOP_LOSS = 25 # <--- Governs when we give up on a stream/horizon
# Get historical data, fit a copula, and submit
def fit_and_sample(lagged_zvalues: [[float]], num: int, copula=None):
""" Example of creating a "sample" of future values
lagged_zvalues: [ [z1,z2,z3] ] distributed N(0,1) margins, roughly
copula : Something from https://pypi.org/project/copulas/
returns: [ [z1, z2, z3] ] representative sample
Swap out this function for whatever you like.
from microprediction import MicroWriter
import json
import os
# Microprediction writer
write_key = os.environ.get(
'WRITE_KEY'
) # GitHub action needs to set env variable. You need to create a GitHub secret called WRITE_KEY
if write_key is None:
raise Exception(
'The write key was not injected into the environment ... this needs to be done by github actions perhaps'
)
print(write_key)
mw = MicroWriter(write_key=write_key)
assert mw.key_difficulty(
mw.write_key) >= 12, "You need a key of difficulty 12 to create a stream"
print(mw.animal)
# Chess.com
URL_TEMPLATE = 'https://api.chess.com/pub/player/HANDLE/stats'
HOME_TEMPLATE = 'https://www.chess.com/member/HANDLE'
# Microprediction.Org
STREAM_URL_TEMPLATE = 'https://www.microprediction.org/stream_dashboard.html?stream=CATEGORY_TYPE_HANDLE'
STREAM_TYPES = ['daily', 'level', 'change']
ANALYSIS_DIR = 'analysis'
def stream_url(category, stream_type, handle):
return STREAM_URL_TEMPLATE.replace('CATEGORY', category).replace(
'TYPE', stream_type).replace('HANDLE', handle)
from credentials import ELFEST_BOBCAT # You'll have to supply your own
import pandas as pd
from microprediction import MicroWriter
import time
# Video explanation of this example:
# https://vimeo.com/443203883
mw = MicroWriter(write_key=ELFEST_BOBCAT) # See creating_a_key.py
STREAM_NAME = 'water.json'
def water_height():
df = pd.read_csv('https://www.ndbc.noaa.gov/data/realtime2/21413.dart')
return float(df.iloc[1, :].values[0].split(' ')[-1])
def poll_for_an_hour():
for _ in range(4):
mw.set(name=STREAM_NAME, value=water_height())
time.sleep(15 * 60)
print('.', flush=True)
if __name__ == '__main__':
poll_for_an_hour()
from microprediction import new_key, MicroWriter import pprint write_key = "ddf3b83838f7d07f0e48404115eb3ec3" or new_key(difficulty=8) mw = MicroWriter(write_key=write_key) name = 'cop.json' values = [float(x) / 10.1 for x in range(mw.num_predictions)] res = mw.submit(name=name, values=values, delay=mw.delays[0]) pprint.pprint(res)
from microprediction import MicroWriter, new_key
import random
from pprint import pprint
from collections import Counter
# Crush the die.json contest and some others using the new discrete_pdf_lagged functionality
NAMES = ['die.json']
if __name__ == "__main__":
write_key = new_key(difficulty=8)
print(write_key)
mw = MicroWriter(write_key=write_key, base_url='https://devapi.microprediction.org')
for name in NAMES:
for delay in mw.DELAYS:
lagged_values = mw.get_lagged_values(name=name)
pdf = mw.get_discrete_pdf_lagged(name=name, delay=mw.DELAYS[1],
lagged_values=lagged_values) # "market measure"
empirical_pdf = Counter(lagged_values)
market_pdf = dict(zip(pdf['x'], pdf['y']))
weights = [empirical_pdf[x] / (0.01 + market_pdf[x]) for x in pdf['x']]
samples = random.choices(population=pdf['x'], weights=weights, k=mw.num_predictions)
res = mw.submit(name=name, delay=delay, values=samples, verbose=True)
pprint(res)
print('Punch ' + write_key + ' into dashboard at https://www.microprediction.org/dashboard.html')
