data = get_data()
s = data[0]
x, y_fof2, y_mufd, y_hmf2, sigma = [], [], [], [], []
first_tm = None
last_tm = None
for pt in s['history']:
tm = (pd.to_datetime(pt[0]) -
pd.Timestamp("1970-01-01")) // pd.Timedelta("1s")
cs = pt[1]
if cs < 10 and cs != -1:
continue
sd = cs_to_stdev(cs, adj100=True)
fof2, mufd, hmf2 = pt[2:5]
print("%d\t%f\t%f\t%f\t%f\t%d" % (tm, fof2, mufd, hmf2, sd, cs))
x.append(tm / 86400.)
y_fof2.append(np.log(fof2))
y_mufd.append(np.log(mufd))
y_hmf2.append(np.log(hmf2))
sigma.append(sd)
if first_tm is None:
first_tm = tm
last_tm = tm
print("")
x = np.array(x)
def generate():
dsn = "dbname='%s' user='******' host='%s' password='******'" % (
os.getenv("DB_NAME"), os.getenv("DB_USER"), os.getenv("DB_HOST"),
os.getenv("DB_PASSWORD"))
con = psycopg2.connect(dsn)
times = [
dt.datetime.fromtimestamp(float(ts))
for ts in request.form.getlist('target')
]
if len(times) == 0:
times = [dt.datetime.utcnow()]
run_id = int(request.form.get('run_id', -1))
data = get_data('http://localhost:%s/history.json?days=14' %
(os.getenv('HISTORY_PORT')))
out = []
for station in data:
x, y_fof2, y_mufd, y_hmf2, sigma = [], [], [], [], []
for pt in station['history']:
tm = (pd.to_datetime(pt[0]) -
pd.Timestamp("1970-01-01")) // pd.Timedelta("1s")
cs = pt[1]
if cs < 10 and cs != -1:
continue
sd = cs_to_stdev(cs, adj100=True)
fof2, mufd, hmf2 = pt[2:5]
x.append(tm / 86400.)
y_fof2.append(np.log(fof2))
y_mufd.append(np.log(mufd))
y_hmf2.append(np.log(hmf2))
sigma.append(sd)
if len(x) < 7:
continue
x = np.array(x)
y_fof2 = np.array(y_fof2)
mean_fof2 = np.mean(y_fof2)
y_fof2 -= mean_fof2
y_mufd = np.array(y_mufd)
mean_mufd = np.mean(y_mufd)
y_mufd -= mean_mufd
y_hmf2 = np.array(y_hmf2)
mean_hmf2 = np.mean(y_hmf2)
y_hmf2 -= mean_hmf2
sigma = np.array(sigma)
# gp = george.GP(kernel, solver=george.HODLRSolver, tol=1e-5)
gp = george.GP(kernel)
gp.compute(x, sigma + 1e-3)
tm = np.array([time.mktime(ts.timetuple()) for ts in times])
pred_fof2, sd_fof2 = gp.predict(y_fof2, tm / 86400., return_var=True)
pred_fof2 += mean_fof2
sd_fof2 = sd_fof2**0.5
pred_mufd, sd_mufd = gp.predict(y_mufd, tm / 86400., return_var=True)
pred_mufd += mean_mufd
sd_mufd = sd_mufd**0.5
pred_hmf2, sd_hmf2 = gp.predict(y_hmf2, tm / 86400., return_var=True)
pred_hmf2 += mean_hmf2
sd_hmf2 = sd_hmf2**0.5
for i in range(len(times)):
with con.cursor() as cur:
cur.execute(
"""
insert into prediction (run_id, station_id, time, cs, log_stdev, fof2, mufd, hmf2)
values (%s, %s, %s, %s, %s, %s, %s, %s)
on conflict (station_id, run_id, time) do update
set cs=excluded.cs, log_stdev=excluded.log_stdev,
fof2=excluded.fof2, mufd=excluded.mufd, hmf2=excluded.hmf2
""", (
run_id,
station['id'],
times[i],
stdev_to_cs(sd_mufd[i]),
sd_mufd[i],
np.exp(pred_fof2[i]),
np.exp(pred_mufd[i]),
np.exp(pred_hmf2[i]),
))
con.commit()
return make_response("OK")
def make_dataset(station):
url = 'http://localhost:5502/mixscale.json?station=%d&points=%d&max_span=120' % (
station, points_per_station)
data = get_data(url)
s = data[0]
x, y_fof2, y_mufd, y_hmf2, sigma, cslist = [], [], [], [], [], []
last_tm = None
iri = subprocess.Popen(
['/build/iri_opt'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
universal_newlines=True,
)
for pt in s['history']:
tm = (pd.to_datetime(pt[0]) -
pd.Timestamp("1970-01-01")) // pd.Timedelta("1s")
cs = pt[1]
iritime = pd.to_datetime(pt[0]).strftime('%Y %m %d %H %M %S')
if cs < 10 and cs != -1:
continue
sd = cs_to_stdev(cs, adj100=True)
fof2, mufd, hmf2 = pt[2:5]
iri.stdin.write("%f %f %f %s\n" %
(s['latitude'], s['longitude'], -100.0, iritime))
iri.stdin.flush()
iridata = [float(x) for x in iri.stdout.readline().split()]
iri_fof2, iri_mufd, iri_hmf2 = iridata[3], iridata[5], iridata[6]
x.append(tm / 86400.)
y_fof2.append(np.log(fof2) - np.log(iri_fof2))
y_mufd.append(np.log(mufd) - np.log(iri_mufd))
y_hmf2.append(np.log(hmf2) - np.log(iri_hmf2))
sigma.append(sd)
cslist.append(cs)
last_tm = tm
iri.stdin.close()
iri.wait()
ds = {}
ds['station_id'] = s['id']
ds['latitude'] = s['latitude']
ds['longitude'] = s['longitude']
ds['x'] = np.array(x)
ds['last_tm'] = last_tm
ds['y_fof2'] = np.array(y_fof2)
ds['mean_fof2'] = np.mean(ds['y_fof2'])
ds['y_fof2'] -= ds['mean_fof2']
ds['y_mufd'] = np.array(y_mufd)
ds['mean_mufd'] = np.mean(ds['y_mufd'])
ds['y_mufd'] -= ds['mean_mufd']
ds['y_hmf2'] = np.array(y_hmf2)
ds['mean_hmf2'] = np.mean(ds['y_hmf2'])
ds['y_hmf2'] -= ds['mean_hmf2']
ds['sigma'] = np.array(sigma)
ds['cs'] = np.array(cslist)
ds['gp'] = george.GP(delta_kernel)
ds['gp'].compute(ds['x'], ds['sigma'] + 1e-3)
return ds
def get_datasets():
data = get_data('http://localhost:%s/pred_sample.json?samples=%d' % (os.getenv('API_PORT'), NSAMPLES))
ds = None
dss = []
for row in data:
if ds is not None and row['run_id'] != ds['run_id']:
dss.append(ds)
ds = None
if ds is None:
ds = { 'run_id': row['run_id'], 'measurements': [] }
station = stations.get(row['station_name'])
if station is None:
continue
x, y, z = sph_to_xyz(station['latitude'], station['longitude'])
meas = {
'lat': station['latitude'],
'lon': station['longitude'],
'x': x,
'y': y,
'z': z,
'fof2': row['fof2'],
'hmf2': row['hmf2'],
'mufd': row['mufd'],
'cs': row['cs'],
'sigma': cs_to_stdev(row['cs']),
}
ds['measurements'].append(meas)
if ds is not None and len(ds['measurements']) > 0:
dss.append(ds)
for ds in dss:
x = np.array([ meas['x'] for meas in ds['measurements'] ])
y = np.array([ meas['y'] for meas in ds['measurements'] ])
z = np.array([ meas['z'] for meas in ds['measurements'] ])
cs = np.array([ meas['cs'] for meas in ds['measurements'] ])
sigma = np.array([ meas['sigma'] for meas in ds['measurements'] ])
fof2 = np.array([ np.log(meas['fof2']) for meas in ds['measurements'] ])
mean_fof2 = np.mean(fof2)
fof2 -= mean_fof2
hmf2 = np.array([ np.log(meas['hmf2']) for meas in ds['measurements'] ])
mean_hmf2 = np.mean(hmf2)
hmf2 -= mean_hmf2
mufd = np.array([ np.log(meas['mufd']) for meas in ds['measurements'] ])
mean_mufd = np.mean(mufd)
mufd -= mean_mufd
ds['arrays'] = {
'xyz': np.column_stack((x,y,z)),
'cs': cs,
'sigma': sigma,
'fof2': fof2,
'mean_fof2': mean_fof2,
'hmf2': hmf2,
'mean_hmf2': mean_hmf2,
'mufd': mufd,
'mean_mufd': mean_mufd,
}
ds['gp'] = george.GP(kernel)
ds['gp'].compute(ds['arrays']['xyz'], ds['arrays']['sigma'] + 1e-3)
return dss
def generate():
dsn = "dbname='%s' user='******' host='%s' password='******'" % (
os.getenv("DB_NAME"), os.getenv("DB_USER"), os.getenv("DB_HOST"),
os.getenv("DB_PASSWORD"))
con = psycopg2.connect(dsn)
times = [
dt.datetime.fromtimestamp(float(ts))
for ts in request.form.getlist('target')
]
if len(times) == 0:
times = [dt.datetime.utcnow()]
run_id = int(request.form.get('run_id', -1))
data = get_data('http://localhost:%s/history.json?days=14' %
(os.getenv('HISTORY_PORT')))
with con.cursor() as cur:
cur.execute(
'select ssn from essn where run_id=%s and series=%s order by time desc nulls last limit 1',
(run_id, '24h'))
(ssn, ) = cur.fetchone()
out = []
for station in data:
x, y_fof2, y_mufd, y_hmf2, sigma = [], [], [], [], []
for pt in station['history']:
ts = pd.to_datetime(pt[0])
tm = (ts - pd.Timestamp("1970-01-01")) // pd.Timedelta("1s")
cs = pt[1]
if cs < 10 and cs != -1:
continue
sd = cs_to_stdev(cs, adj100=True)
fof2, mufd, hmf2 = pt[2:5]
iridata = get_iri(station['latitude'], station['longitude'], ssn,
ts)
iri_fof2, iri_mufd, iri_hmf2 = iridata[3], iridata[5], iridata[6]
x.append(tm / 86400.)
y_fof2.append(np.log(fof2) - np.log(iri_fof2))
y_mufd.append(np.log(mufd) - np.log(iri_mufd))
y_hmf2.append(np.log(hmf2) - np.log(iri_hmf2))
sigma.append(sd)
if len(x) < 7:
continue
x = np.array(x)
y_fof2 = np.array(y_fof2)
mean_fof2 = np.mean(y_fof2)
y_fof2 -= mean_fof2
y_mufd = np.array(y_mufd)
mean_mufd = np.mean(y_mufd)
y_mufd -= mean_mufd
y_hmf2 = np.array(y_hmf2)
mean_hmf2 = np.mean(y_hmf2)
y_hmf2 -= mean_hmf2
sigma = np.array(sigma)
gp = george.GP(delta_kernel)
gp.compute(x, sigma + 1e-3)
tm = np.array([time.mktime(ts.timetuple()) for ts in times])
pred_fof2, sd_fof2 = gp.predict(y_fof2, tm / 86400., return_var=True)
pred_fof2 += mean_fof2
sd_fof2 = sd_fof2**0.5
pred_mufd, sd_mufd = gp.predict(y_mufd, tm / 86400., return_var=True)
pred_mufd += mean_mufd
sd_mufd = sd_mufd**0.5
pred_hmf2, sd_hmf2 = gp.predict(y_hmf2, tm / 86400., return_var=True)
pred_hmf2 += mean_hmf2
sd_hmf2 = sd_hmf2**0.5
for i in range(len(times)):
iridata = get_iri(station['latitude'], station['longitude'], ssn,
times[i])
iri_fof2, iri_mufd, iri_hmf2 = iridata[3], iridata[5], iridata[6]
fof2 = np.exp(np.log(iri_fof2) + pred_fof2[i])
mufd = np.exp(np.log(iri_mufd) + pred_mufd[i])
hmf2 = np.exp(np.log(iri_hmf2) + pred_hmf2[i])
delta_fof2 = fof2 - iri_fof2
delta_mufd = mufd - iri_mufd
delta_hmf2 = hmf2 - iri_hmf2
with con.cursor() as cur:
cur.execute(
"""
insert into prediction (run_id, station_id, time, cs, log_stdev, fof2, mufd, hmf2)
values (%s, %s, %s, %s, %s, %s, %s, %s)
on conflict (station_id, run_id, time) do update
set cs=excluded.cs, log_stdev=excluded.log_stdev,
fof2=excluded.fof2, mufd=excluded.mufd, hmf2=excluded.hmf2
""", (
run_id,
station['id'],
times[i],
stdev_to_cs(sd_mufd[i]),
sd_mufd[i],
fof2,
mufd,
hmf2,
))
with con.cursor() as cur:
cur.execute(
"""
insert into prediction_delta (run_id, station_id, time, cs, log_stdev, delta_fof2, delta_mufd, delta_hmf2)
values (%s, %s, %s, %s, %s, %s, %s, %s)
on conflict (station_id, run_id, time) do update
set cs=excluded.cs, log_stdev=excluded.log_stdev,
delta_fof2=excluded.delta_fof2, delta_mufd=excluded.delta_mufd, delta_hmf2=excluded.delta_hmf2
""", (
run_id,
station['id'],
times[i],
stdev_to_cs(sd_mufd[i]),
sd_mufd[i],
delta_fof2,
delta_mufd,
delta_hmf2,
))
con.commit()
return make_response("OK")