def run(self):
while True:
if self.queue.empty():
break
item = self.queue.get()
time.sleep(random.random())
getjson(item)
self.queue.task_done()
def get_fake_live_data():
""" Get 'live' data from a replay of badminton game """
k = int((time.time() - ORIGIN) / (INTERVAL * 60))
url = TEMPLATE_URL.replace('FRAME', str(k).zfill(num_digits))
data = getjson(url=url)
person = data['people'][0]
values = [0.01 * (v - 200) for v in person['pose_keypoints_2d'][3:5]]
return values
def __init__(self,
base_url=None,
num_predictions=None,
min_len=None,
min_balance=None,
delays=None):
""" Establish connection and adopt configuration parameters from site, if not provided """
self.base_url = base_url or default_url()
default_config = getjson(
url='https://config.microprediction.org/config.json',
failover_url='https://stableconfig.microprediction.org/config.json'
)
self.delays = delays or default_config["delays"]
self.num_predictions = num_predictions or default_config[
"num_predictions"]
self.min_len = min_len or default_config["min_len"]
self.min_balance = min_balance or default_config["min_balance"]
def fit(self, name, lagged_values, lagged_times=None, **ignored):
""" Fit one stream, but only if the distribution machine supplied has a fit method """
# Might use offline fit
changed = False
if name not in self.stream_state:
self.stream_state.update({
name:
self.initial_state(name=name,
lagged_values=lagged_values,
lagged_times=lagged_times)
})
else:
machine = self.stream_state[name]['machine']
stored_params = None
if self.param_base_url is not None:
url = self.param_base_url + '/' + name
try:
stored_params = getjson(url)
if stored_params is None:
machine.params = stored_params
except:
print('Failed to get params from ' + url)
if stored_params is None:
try:
changed = machine.fit(lagged_values=lagged_values,
lagged_times=lagged_times)
if changed:
print('Found better params for ' + name)
pprint(machine.params)
print(' ', flush=True)
except AttributeError:
print('Keeping the same params for ' + name)
pprint(machine.params)
print(' ', flush=True)
return changed
write_key = os.environ.get(
'WRITE_KEY') # GitHub action needs to set env variable. You need to create a GitHub secret called WRITE_KEY
mw = MicroWriter(write_key=write_key)
assert mw.key_difficulty(mw.write_key) >= 12, "You need a key of difficulty 12 to create a stream"
CATEGORIES = ['stargazers_count', 'forks_count']
REPOS = {'pykalman': 'https://github.com/pykalman/pykalman',
'microprediction': 'https://github.com/microprediction/microprediction',
'prophet': 'https://github.com/facebook/prophet',
'neuralprophet': 'https://github.com/ourownstory/neural_prophet',
'river': 'https://github.com/online-ml/river',
'scikit-learn': 'https://github.com/scikit-learn/scikit-learn',
'tensorflow': 'https://github.com/tensorflow/tensorflow',
'keras': 'https://github.com/keras-team/keras'}
REPO_APIS = dict( [ (k, v.replace('https://github.com/', 'https://api.github.com/repos/')) for k, v in REPOS.items() ] )
if __name__ == '__main__':
for repo_name, repo_url in REPO_APIS.items():
data = getjson(repo_url)
if data is not None:
for category in CATEGORIES:
current_value = int(data[category])
level_name = 'github_' + category + '_' + repo_name + '.json'
print(mw.set(name=level_name, value=current_value))
else:
print('Something is wrong')
print(repo_url)
try:
from microstate.redis_env_private import NOTHING_MUCH
except ImportError:
# Environment variables need to be set some other way, for example as github secrets
pass
state_config_url = os.getenv('REDIS_CONFIG_URL')
if state_config_url is None:
raise Exception('Cannot get environment variable REDIS_CONFIG_URL')
state_config_failover_url = os.getenv('REDIS_CONFIG_FAILOVER_URL')
if state_config_url is None:
raise Exception(
'Cannot get environment variable REDIS_CONFIG_FAILOVER_URL')
REDIS_CONFIG = getjson(url=state_config_url,
failover_url=state_config_failover_url)
state_test_config_url = os.getenv('REDIS_TEST_CONFIG_URL')
if state_config_url is None:
raise Exception('Cannot get environment variable REDIS_TEST_CONFIG_URL')
state_test_config_failover_url = os.getenv('REDIS_TEST_CONFIG_FAILOVER_URL')
if state_config_url is None:
raise Exception(
'Cannot get environment variable REDIS_TEST_CONFIG_FAILOVER_URL')
REDIS_TEST_CONFIG = getjson(url=state_test_config_url,
failover_url=state_config_failover_url)
if REDIS_CONFIG is None:
raise Exception('Could not get configuration')
def test_getjson():
data = getjson('https://config.microprediction.org/config.json')
assert isinstance(data, dict)
# Get the monthly winners
from getjson import getjson
from rediz.client import Rediz
from rediz.collider_config_private import REDIZ_COLLIDER_CONFIG, EMBLOSSOM_MOTH
from pprint import pprint
if __name__ == '__main__':
rdz = Rediz(**REDIZ_COLLIDER_CONFIG)
sponsor = rdz.shash(EMBLOSSOM_MOTH)
write_key = EMBLOSSOM_MOTH
prizes = getjson('https://api.microprediction.org/prizes/')
lb = rdz.get_regular_monthly_sponsored_leaderboard(sponsor=sponsor)
pprint(lb)
rdz.delete_regular_monthly_sponsored_leaderboard(write_key=write_key)
lb = rdz.get_regular_monthly_sponsored_leaderboard(sponsor=sponsor)
pprint(lb)
rdz.delete_bivariate_monthly_sponsored_leaderboard(
write_key=EMBLOSSOM_MOTH)
rdz.delete_trivariate_monthly_sponsored_leaderboard(
write_key=EMBLOSSOM_MOTH)
def test_getjson_invalidalso_valid():
try:
data = getjson(url=invalid_url, failover_url=valid_url_1)
raise NotImplementedError
except:
pass
def test_getjson_valid_valid():
data = getjson(url=valid_url_1, failover_url=invalid_json_url_1)
assert isinstance(data, dict)
def tasks(host_uri):
return list(getjson(host_uri + "/_active_tasks"))
def docs(url): return int(getjson(url)['doc_count'])
def docs(url):
return int(getjson(url)['doc_count'])
from getjson import getjson
from microprediction import MicroWriter
from config import ACTIVE, mw, URL_TEMPLATE
if __name__ == '__main__':
for category in ACTIVE.keys():
for handle, player in ACTIVE[category].items():
url = URL_TEMPLATE.replace('HANDLE', handle)
data = getjson(url)
try:
value = data[category]['last']['rating']
name = category + '_daily_' + handle + '.json'
print( (name, value, mw.set(name=name,value=value) ) )
except:
print(data)
def get_config():
return getjson(CONFIG_URL)
def get_timing():
return getjson(TIMING_URL)
def get_elo_leaderboard(category='classic',n_dim=5, n_trials=130):
return getjson(elo_leaderboard_url(category=category,n_dim=n_dim,n_trials=n_trials))
import os
from sys import platform
from getjson import getjson
# Retrieve credentials for database where state is stored
try:
from microprediction.set_env_private import NOTHING_MUCH
except ImportError:
# Environment variables need to be set some other way, for example by retrieving github secrets
pass
micro_config_url = os.getenv('MICRO_TEST_CONFIG_URL')
if micro_config_url is None:
raise Exception('Cannot get environment variable MICRO_TEST_CONFIG_URL')
micro_config_failover_url = os.getenv('MICRO_TEST_CONFIG_FAILOVER_URL')
if micro_config_failover_url is None:
raise Exception(
'Cannot get environment variable MICRO_TEST_CONFIG_FAILOVER_URL')
MICRO_TEST_CONFIG = getjson(url=micro_config_url,
failover_url=micro_config_failover_url)
def get_timing():
return getjson(
'https://raw.githubusercontent.com/microprediction/humpday/main/humpday/comparison/timing.json'
)
def get_benchmarks():
return getjson(
'https://raw.githubusercontent.com/microprediction/humpday/main/humpday/comparison/bencharks.json'
)
import os
from sys import platform
from getjson import getjson
from rediz.conventions import MICRO_CONVENTIONS_ARGS
# Load a couple of environment variables for local testing.
# These are secrets on github.
try:
from rediz.rediz_test_config_private import NOTHING_MUCH
except:
pass
test_config_url = os.getenv('TEST_CONFIG_URL')
test_config_failover_url = os.getenv('TEST_CONFIG_FAILOVER_URL')
REDIZ_TEST_CONFIG = getjson(url=test_config_url,
failover_url=test_config_failover_url)
if REDIZ_TEST_CONFIG is None:
raise Exception('Could not get configuration')
if platform == 'darwin':
REDIZ_TEST_CONFIG['DUMP'] = True
REDIZ_FAKE_CONFIG = REDIZ_TEST_CONFIG['FAKE']
for arg in MICRO_CONVENTIONS_ARGS:
if arg not in REDIZ_FAKE_CONFIG:
raise Exception('Need to update REDIZ_FAKE_CONFIG to include ' + arg)
from getjson import getjson
from pprint import pprint
import operator
def descend(d):
return {k: v for k, v in sorted(d.items(), key=lambda item: item[1], reverse=True)}
PRIZES = 'https://api.microprediction.org/prizes/'
pprint([list(zip(money, descend(getjson(url)))) for url, money in getjson(PRIZES).items()])
def test_almost():
data = getjson(url=almost_url_2, failover_url=valid_url_1)
assert isinstance(data, dict)
def test_getjson():
start_time = time.time()
for k in range(5):
getjson('https://config.microprediction.org/config.json')
end_time = time.time()
assert end_time-start_time<2
def tasks(host_uri): return list(getjson(host_uri + "/_active_tasks"))
