def test_cancel_live(): write_key = new_key(difficulty=7) predictor = MicroWriter(write_key=write_key) values = sorted(np.random.randn(predictor.num_predictions)) predictor.submit(name='cop.json', values=values, delay=predictor.DELAYS[0]) res = predictor.cancel(name='cop.json') assert res, "Cancellation of submission failed"
def test_is_valid(): key = "4c3db49d3a291acad7bdedecfa787891" assert MicroWriter.is_valid_key(key) key = b"4c3db49d3a291acad7bdedecfa787891" assert MicroWriter.is_valid_key(key) == False, "Binary keys are not valid" key = "3c3db49d3a291acad7bdedecfa787891" assert not MicroWriter.is_valid_key(key)
def run(): try: mw = MicroWriter(write_key=TRAFFIC_WRITE_KEY) except: raise Exception( "You need to set the write key for this example to work") while True: value = hospital_bike_activity(station_ids) utc_now = pytz.utc.localize(datetime.utcnow()) pst_now = utc_now.astimezone(pytz.timezone("America/Los_Angeles")) res = mw.set(name=NAME, value=float(value)) pprint({ 'PST time': pst_now.strftime("%H:%M"), 'activity': value, "res": res }) print('', flush=True)
def test_repo_live(): if DODDLE_MAMMAL is not None: mw = MicroWriter(write_key=DODDLE_MAMMAL, base_url='https://devapi.microprediction.org') saved_url = mw.get_own_repository() res = mw.set_repository( url= 'https://www.linkedin.com/in/petercotton/detail/recent-activity/posts/' ) print(mw.get_own_repository()) print(res) res = mw.delete_repository() mw.set_repository(url=saved_url) saved_back = mw.get_own_repository() assert saved_back == saved_url
def example_z2_prediction(write_key, name=None): """ :param write_key str See above for creating a write_key :param name str Name of a bivariate stream, such as 'z2~helicopter_psi~helicopter_theta~70.json' """ if name is None: name = 'z2~helicopter_psi~helicopter_theta~70.json' copula = fit_copula_to_z2_data() submitted_points = copula.sample(mr.num_predictions) qpitch = submitted_points[0] qyaw = submitted_points[1] values = sorted([project(p, q) for p, q in zip(qpitch, qyaw)]) # Now submit them mw = MicroWriter(write_key=write_key) print(mw.animal + ' is submitting ') horizon = 70 reply = mw.submit(name=name, values=values, delay=horizon) pprint(reply)
def run(): try: mw = MicroWriter(write_key=TRAFFIC_WRITE_KEY) except: raise Exception( "You need to set the write key for this example to work") while True: value = number_of_articles_that_mention_cuomo_or_nyc() utc_now = pytz.utc.localize(datetime.utcnow()) pst_now = utc_now.astimezone(pytz.timezone("America/Los_Angeles")) try: res = mw.set(name=NAME, value=float(value)) pprint({ 'PST time': pst_now.strftime("%H:%M"), 'count': value, "res": res }) print('', flush=True) except: continue
# # Work in progress ... broken currently # # Example of sending live data to www.microprediction.com to predict a simulated sequence # We generate a time series of the number of people who are infected in a model for an epidemic # that is similar to CIR but takes novelty of interaction into account. see https://www.overleaf.com/read/sgjvfxydcwpk # The time series comprises a sequence of daily observations (real time) updated every 90 seconds. NAMES = ['susceptible', 'infected', 'recovered'] DT = 90 # Seconds between each update try: from microprediction.config_private import MATCHABLE_BAT mw = MicroWriter(write_key=MATCHABLE_BAT) except: raise Exception( "You need to set the write key for this example to work, and it must be 'copula strength' " ) def entire(a, t): if a * t > 1e-4: return (1 - math.exp(-a * t)) / (a * t) else: return 1.0 def compartmental(t, y, a, b, gamm): """
def test_new(): write_key = new_key(difficulty=7) assert MicroWriter.is_valid_key(write_key)
############################################################## # parameters and details credentials = Credentials(username=os.getenv('username'), password=os.getenv('password')) client = AmphoraDataRepositoryClient(credentials) Electricity_Amphora_id = '89c2e30d-78c8-46ef-b591-140edd84ddb6' Weather_Amphora_id = '2866d34f-fe2b-4c07-acdd-28fe2d6d9194' Forecast_amphora_id = '78831a6b-de49-454e-9e5c-7c757478e783' name = 'South_Australia_Electricity_Price.json' ############################################################# # Publish raw data on microprediction mw = MicroWriter(write_key="bdfd44affd28e6c5b45329d6d4df7729") time_range = a10a.DateTimeRange(_from=datetime.utcnow() + timedelta(hours=-1), to=datetime.utcnow()) electricity_amphora = client.get_amphora(Electricity_Amphora_id) electricity_signals = electricity_amphora.get_signals() df = electricity_signals.pull(date_time_range=time_range).to_pandas() price = df['price'] mw.set(name=name, value=price[-1]) ########################################################## # Create a prediction and publish on microprediction # length = mw.num_predictions # data_time_range = a10a.DateTimeRange(_from = datetime.utcnow() + timedelta(hours=-length/2) , to= datetime.utcnow() ) # pdf = electricity_signals.pull(date_time_range=data_time_range).to_pandas() # price = pdf['price'] # prices_of_interest = price[-length-1:-1]
# get started creating streams (see the 4th module in particular) import logging import urllib import time import pytz import random from datetime import datetime from pprint import pprint from microprediction import MicroWriter import os write_key = os.environ.get( 'WRITE_KEY' ) # GitHub action needs to set env variable. You need to create a GitHub secret called WRITE_KEY mw = MicroWriter(write_key=write_key) assert mw.key_difficulty( mw.write_key) >= 13, "You need a key of difficulty 13 for copula streams" mw.set_repository( url= 'https://github.com/microprediction/microprediction/blob/master/microprediction/live/seattle_wind.py' ) # courtesy logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) def wait_between_attempts(): """ Incremental backoff between connection attempts """ wait_time = 5.3 # time is in seconds while True:
#!/usr/bin/python3.8 from microprediction import MicroWriter from apscheduler.schedulers.blocking import BlockingScheduler import numpy as np from pprint import pprint from scipy.integrate import odeint from math import sqrt # Example of sending live data to www.microprediction.org and kick-starting copula streams # This uses the MicroWriter directly, though soon there will be a parent class that makes this more convenient NAMES = ['three_body_x.json', 'three_body_y.json', 'three_body_z.json'] try: from microprediction.config_private import THREE_BODY_WRITE_KEY mw = MicroWriter(write_key=THREE_BODY_WRITE_KEY) except: raise Exception( "You need to set the write key for this example to work, and it must be 'copula strength' " ) SCALE = 10**8 Y0 = [384.6 * 10**6, 0, 0, 1197000, -9928000, 0, 0, 1025, 0, 8490, -2455, 100] print("Initial value is " + str([y / SCALE for y in Y0[:3]]), flush=True) Y = Y0 def threebody(Y, t): """ Forked from https://guillaumecantin.pythonanywhere.com/animation/4/ """ m1 = 5.974 * 10**24 # Earth
from microprediction import MicroWriter import time from pprint import pprint import numpy as np # This example illustrates how easy it is to enter z1~ streams # For a video explanation of z1 streams and see https://www.linkedin.com/feed/update/urn:li:activity:6696569137365430272/ # See https://www.linkedin.com/pulse/short-introduction-z-streams-peter-cotton-phd/ for an explanation of z-streams. # You need to set a write key. # Supplying a URL to your repository is optional. # This will do is make your position on the leaderboards clickable and direct people to your work. # You can alternatively provide your personal homepage or LinkedIn profile WRITE_KEY = DODDLE_MAMMAL ANIMAL = MicroWriter.animal_from_key(DODDLE_MAMMAL) REPO = 'https://github.com/microprediction/microprediction/blob/master/crawler_alternatives/' + ANIMAL.lower( ).replace(' ', '_') + '.py' # We simply loop over all z1~ streams # These streams are approximately normally distributed. # We enter N(0,1) distributed samples so as to try to take advantage of algorithms # which accidentally submit distributional predictions far from this. # This code could be run as a cron job or scheduled task at PythonAnywhere, for example def jiggle(): return 0.001 * np.random.randn() if __name__ == "__main__":
from microprediction import new_key, MicroWriter write_key = new_key(difficulty=9) mw = MicroWriter(write_key=write_key) die_rolls = [k - 2.5 for k in range(6)] * 50 values = die_rolls[:mw.num_predictions] res = mw.submit(name='die.json', values=values) print('https://api.microprediction.org/home/' + write_key)
#!/usr/bin/python3.8 from microprediction import MicroWriter from microprediction.live import bronx_speed from apscheduler.schedulers.blocking import BlockingScheduler from pprint import pprint import datetime # Example of sending live data change to www.microprediction.com try: from microprediction.config_private import TRAFFIC_WRITE_KEY mw = MicroWriter(write_key=TRAFFIC_WRITE_KEY) except: raise Exception("You need to set the write key for this example to work") NAME = "bronx_traffic_speed_on_change.json" prev_speed = None def poll_and_send_if_changed(): """ Modify feed of speed changes - or at least keep it warm """ global speed, prev_speed if prev_speed is None: prev_speed = bronx_speed() speed = None else: speed = bronx_speed() try: fspeed = float(speed) fprev = float(prev_speed)
def test_client(): with Timer() as t: write_key = new_key(difficulty=7) mw = MicroWriter(write_key=write_key) assert (t.elapsed < 15)
from microprediction import MicroWriter import os import random SECONDS = 60*15 # How long to spend making a throwaway key if __name__=='__main__': # WRITE_KEY is injected into environ by GitHub action. Grab it. write_key = os.environ.get('WRITE_KEY') if ',' in write_key: write_keys = write_key.split(',') write_key = random.choice(write_keys) # Create a throwaway key and use it to bolster the balance, using a transfer. if write_key is not None: mw = MicroWriter(write_key=write_key) balance = float(mw.get_balance()) if balance<0: print('Trying to top up balance for '+mw.animal, ' which is currently '+str(balance)) throwaway_key = mw.maybe_create_key(difficulty=11, seconds=SECONDS) if throwaway_key is None: print('Sorry, no luck this time') else: print('Hit the jackpot ... topping up balance for '+ mw.animal) mw.put_balance(source_write_key=throwaway_key, amount=256) print('New balance is '+str(mw.get_balance())) else: print('Balance for '+mw.animal+' is '+str(balance)+' so no need to top up.')
# 1. Grab the Github secrets try: # For this script to work you need to create four separate GitHub secrets # called WRITE_KEY_1 WRITE_KEY_2 WRITE_KEY_3 and WRITE_KEY_4 # The idea is that this way you get 900 samples instead of 225 WRITE_KEYS = [os.environ.get('WRITE_KEY_' + str(i + 1)) for i in range(4)] assert len(WRITE_KEYS) == 4, 'Need four write keys to make the syndicate' except: # Or one secret called WRITE_KEY or WRITE_KEYS with them comma separated WRITE_KEYS_comma_sep = os.environ.get('WRITE_KEYS') or os.environ.get( 'WRITE_KEY') WRITE_KEYS = WRITE_KEYS_comma_sep.split(',') print('Copula syndicate is firing up.') for write_key in WRITE_KEYS: animal = MicroWriter.animal_from_key(write_key) print(animal) # 2. Pick a copula VINES = [ 'center', 'regular', 'direct' ] # See https://sdv.dev/Copulas/tutorials/03_Multivariate_Distributions.html#Vine-Copulas VINE_TYPE = random.choice( VINES ) # Perhaps you want to fix this choice. This way we get lots of plots. # 3. (Optional) Set the URL of your repo so that others can learn from it REPO = 'https://github.com/microprediction/microactors-plots/blob/master/fit.py' # <--- Change your username PLOTS_PATH = os.path.dirname( os.path.realpath(__file__)) + os.path.sep + 'gallery'
def test_delays_2(): mc = MicroWriter(write_key=create_key(difficulty=7)) assert len(mc.DELAYS) == 3
import os from getjson import getjson from microprediction import MicroWriter # This code lies at https://github.com/microprediction/chess/blob/main/set.py # and generates streams such as https://www.microprediction.org/stream_dashboard.html?stream=chess_bullet_level_Hikaru write_key = os.environ.get('WRITE_KEY') # GitHub action needs to set env variable. You need to create a GitHub secret called WRITE_KEY mw = MicroWriter(write_key=write_key) assert mw.key_difficulty(mw.write_key)>=12, "You need a key of difficulty 12 to create a stream" PLAYERS = {'Hikaru': 'hikaru_nakamura', 'Firouzja2003': 'alireza_firouzja', 'GMWSO': 'wesley_so', 'LyonBeast': 'maxime_vachier_lagrave', 'nihalsarin': 'nihal_sarin', 'DanielNaroditsky': 'daniel_naroditsky', 'PinIsMightier': 'halloween_gambit'} URL_TEMPLATE = 'https://api.chess.com/pub/player/HANDLE/stats' CATEGORIES = ['chess_blitz', 'chess_bullet'] if __name__ == '__main__': for category in CATEGORIES: for handle, player in PLAYERS.items(): url = URL_TEMPLATE.replace('HANDLE', handle.lower()) data = getjson(url) if data is not None: current_value = int(data[category]['last']['rating']) level_name = category + '_level_' + handle + '.json' # Name of stream with rating level
from microprediction import new_key, MicroWriter from pprint import pprint write_key = new_key(difficulty=9) mw = MicroWriter(write_key=write_key) # New video tutorials are available at https://www.microprediction.com/python-1 to help you # get started running crawlers at www.microprediction.com if __name__ == '__main__': die_rolls = [k - 2.5 for k in range(6)] * 50 values = die_rolls[:mw.num_predictions] pprint(mw.submit(name='die.json', values=values)) print('https://api.microprediction.org/home/' + write_key)
import os from getjson import getjson from microprediction import MicroWriter write_key = os.environ.get( 'WRITE_KEY') # GitHub action needs to set env variable. You need to create a GitHub secret called WRITE_KEY mw = MicroWriter(write_key=write_key) assert mw.key_difficulty(mw.write_key) >= 12, "You need a key of difficulty 12 to create a stream" CATEGORIES = ['stargazers_count', 'forks_count'] REPOS = {'pykalman': 'https://github.com/pykalman/pykalman', 'microprediction': 'https://github.com/microprediction/microprediction', 'prophet': 'https://github.com/facebook/prophet', 'neuralprophet': 'https://github.com/ourownstory/neural_prophet', 'river': 'https://github.com/online-ml/river', 'scikit-learn': 'https://github.com/scikit-learn/scikit-learn', 'tensorflow': 'https://github.com/tensorflow/tensorflow', 'keras': 'https://github.com/keras-team/keras'} REPO_APIS = dict( [ (k, v.replace('https://github.com/', 'https://api.github.com/repos/')) for k, v in REPOS.items() ] ) if __name__ == '__main__': for repo_name, repo_url in REPO_APIS.items(): data = getjson(repo_url) if data is not None: for category in CATEGORIES: current_value = int(data[category]) level_name = 'github_' + category + '_' + repo_name + '.json' print(mw.set(name=level_name, value=current_value)) else: print('Something is wrong')
import numpy as np from pprint import pprint import matplotlib.pyplot as plt import random import time import warnings warnings.filterwarnings('ignore') from copulas.multivariate import GaussianMultivariate import pandas as pd # Grab the Github secret import os Doodle_Fox = os.environ.get('Doodle_Fox') # WRITE_KEY = Doodle_Fox ANIMAL = MicroWriter.animal_from_key(WRITE_KEY) REPO = 'https://github.com/spikeshr/microactors/blob/master/fit.py' print('This is ' + ANIMAL) VERBOSE = False # Get historical data, fit a copula, and submit def fit_and_sample(lagged_zvalues: [[float]], num: int, copula=None): """ Example of fitting a copula function, and sampling lagged_zvalues: [ [z1,z2,z3] ] distributed N(0,1) margins, roughly copula : Something from https://pypi.org/project/copulas/ returns: [ [z1, z2, z3] ] representative sample """ # Remark: It's lazy to just sample synthetic data
from microprediction import MicroWriter from microprediction.samplers import exponential_bootstrap import time from pprint import pprint from microprediction.config_private import OXEATE_MAMMAL # Enters all the z2 streams WRITE_KEY = OXEATE_MAMMAL ANIMAL = MicroWriter.animal_from_key(WRITE_KEY) print(ANIMAL, flush=True) STOP_LOSS = 250. if __name__ == "__main__": mw = MicroWriter(write_key=WRITE_KEY) LONG = '~' + str(mw.DELAYS[-1]) SHORT = '~' + str(mw.DELAYS[0]) # Give up if things are going badly ... do it nicely for _ in range(5): mw.cancel_worst_active(stop_loss=STOP_LOSS, num=1) time.sleep(1) for name in mw.get_streams(): if 'z2~' in name and (LONG in name or SHORT in name): num = mw.num_predictions z11 = '~'.join(['z1', name.split('~')[1], name.split('~')[3]]) z12 = '~'.join(['z1', name.split('~')[2], name.split('~')[3]]) lagged1 = mw.get_lagged_values(z11) lagged2 = mw.get_lagged_values(z12) if len(lagged1) > 10:
from microprediction import MicroWriter import numpy as np from pprint import pprint import matplotlib.pyplot as plt import random import time import warnings warnings.filterwarnings('ignore') from copulas.multivariate import GaussianMultivariate import pandas as pd # Grab the Github secret import os WRITE_KEY = os.environ.get('WRITE_KEY') # <-- You need to add a Github secret ANIMAL = MicroWriter.animal_from_key(WRITE_KEY) # <-- Your nom de plume REPO = 'https://github.com/microprediction/microactors/blob/master/fit.py' # <--- Change your username print('This is ' + ANIMAL + ' firing up') STOP_LOSS = 25 # <--- Governs when we give up on a stream/horizon # Get historical data, fit a copula, and submit def fit_and_sample(lagged_zvalues: [[float]], num: int, copula=None): """ Example of creating a "sample" of future values lagged_zvalues: [ [z1,z2,z3] ] distributed N(0,1) margins, roughly copula : Something from https://pypi.org/project/copulas/ returns: [ [z1, z2, z3] ] representative sample Swap out this function for whatever you like.
from microprediction import MicroWriter import json import os # Microprediction writer write_key = os.environ.get( 'WRITE_KEY' ) # GitHub action needs to set env variable. You need to create a GitHub secret called WRITE_KEY if write_key is None: raise Exception( 'The write key was not injected into the environment ... this needs to be done by github actions perhaps' ) print(write_key) mw = MicroWriter(write_key=write_key) assert mw.key_difficulty( mw.write_key) >= 12, "You need a key of difficulty 12 to create a stream" print(mw.animal) # Chess.com URL_TEMPLATE = 'https://api.chess.com/pub/player/HANDLE/stats' HOME_TEMPLATE = 'https://www.chess.com/member/HANDLE' # Microprediction.Org STREAM_URL_TEMPLATE = 'https://www.microprediction.org/stream_dashboard.html?stream=CATEGORY_TYPE_HANDLE' STREAM_TYPES = ['daily', 'level', 'change'] ANALYSIS_DIR = 'analysis' def stream_url(category, stream_type, handle): return STREAM_URL_TEMPLATE.replace('CATEGORY', category).replace( 'TYPE', stream_type).replace('HANDLE', handle)
from credentials import ELFEST_BOBCAT # You'll have to supply your own import pandas as pd from microprediction import MicroWriter import time # Video explanation of this example: # https://vimeo.com/443203883 mw = MicroWriter(write_key=ELFEST_BOBCAT) # See creating_a_key.py STREAM_NAME = 'water.json' def water_height(): df = pd.read_csv('https://www.ndbc.noaa.gov/data/realtime2/21413.dart') return float(df.iloc[1, :].values[0].split(' ')[-1]) def poll_for_an_hour(): for _ in range(4): mw.set(name=STREAM_NAME, value=water_height()) time.sleep(15 * 60) print('.', flush=True) if __name__ == '__main__': poll_for_an_hour()
from microprediction import new_key, MicroWriter import pprint write_key = "ddf3b83838f7d07f0e48404115eb3ec3" or new_key(difficulty=8) mw = MicroWriter(write_key=write_key) name = 'cop.json' values = [float(x) / 10.1 for x in range(mw.num_predictions)] res = mw.submit(name=name, values=values, delay=mw.delays[0]) pprint.pprint(res)
from microprediction import MicroWriter, new_key import random from pprint import pprint from collections import Counter # Crush the die.json contest and some others using the new discrete_pdf_lagged functionality NAMES = ['die.json'] if __name__ == "__main__": write_key = new_key(difficulty=8) print(write_key) mw = MicroWriter(write_key=write_key, base_url='https://devapi.microprediction.org') for name in NAMES: for delay in mw.DELAYS: lagged_values = mw.get_lagged_values(name=name) pdf = mw.get_discrete_pdf_lagged(name=name, delay=mw.DELAYS[1], lagged_values=lagged_values) # "market measure" empirical_pdf = Counter(lagged_values) market_pdf = dict(zip(pdf['x'], pdf['y'])) weights = [empirical_pdf[x] / (0.01 + market_pdf[x]) for x in pdf['x']] samples = random.choices(population=pdf['x'], weights=weights, k=mw.num_predictions) res = mw.submit(name=name, delay=delay, values=samples, verbose=True) pprint(res) print('Punch ' + write_key + ' into dashboard at https://www.microprediction.org/dashboard.html')