data = get_data() s = data[0] x, y_fof2, y_mufd, y_hmf2, sigma = [], [], [], [], [] first_tm = None last_tm = None for pt in s['history']: tm = (pd.to_datetime(pt[0]) - pd.Timestamp("1970-01-01")) // pd.Timedelta("1s") cs = pt[1] if cs < 10 and cs != -1: continue sd = cs_to_stdev(cs, adj100=True) fof2, mufd, hmf2 = pt[2:5] print("%d\t%f\t%f\t%f\t%f\t%d" % (tm, fof2, mufd, hmf2, sd, cs)) x.append(tm / 86400.) y_fof2.append(np.log(fof2)) y_mufd.append(np.log(mufd)) y_hmf2.append(np.log(hmf2)) sigma.append(sd) if first_tm is None: first_tm = tm last_tm = tm print("") x = np.array(x)
def generate(): dsn = "dbname='%s' user='******' host='%s' password='******'" % ( os.getenv("DB_NAME"), os.getenv("DB_USER"), os.getenv("DB_HOST"), os.getenv("DB_PASSWORD")) con = psycopg2.connect(dsn) times = [ dt.datetime.fromtimestamp(float(ts)) for ts in request.form.getlist('target') ] if len(times) == 0: times = [dt.datetime.utcnow()] run_id = int(request.form.get('run_id', -1)) data = get_data('http://localhost:%s/history.json?days=14' % (os.getenv('HISTORY_PORT'))) out = [] for station in data: x, y_fof2, y_mufd, y_hmf2, sigma = [], [], [], [], [] for pt in station['history']: tm = (pd.to_datetime(pt[0]) - pd.Timestamp("1970-01-01")) // pd.Timedelta("1s") cs = pt[1] if cs < 10 and cs != -1: continue sd = cs_to_stdev(cs, adj100=True) fof2, mufd, hmf2 = pt[2:5] x.append(tm / 86400.) y_fof2.append(np.log(fof2)) y_mufd.append(np.log(mufd)) y_hmf2.append(np.log(hmf2)) sigma.append(sd) if len(x) < 7: continue x = np.array(x) y_fof2 = np.array(y_fof2) mean_fof2 = np.mean(y_fof2) y_fof2 -= mean_fof2 y_mufd = np.array(y_mufd) mean_mufd = np.mean(y_mufd) y_mufd -= mean_mufd y_hmf2 = np.array(y_hmf2) mean_hmf2 = np.mean(y_hmf2) y_hmf2 -= mean_hmf2 sigma = np.array(sigma) # gp = george.GP(kernel, solver=george.HODLRSolver, tol=1e-5) gp = george.GP(kernel) gp.compute(x, sigma + 1e-3) tm = np.array([time.mktime(ts.timetuple()) for ts in times]) pred_fof2, sd_fof2 = gp.predict(y_fof2, tm / 86400., return_var=True) pred_fof2 += mean_fof2 sd_fof2 = sd_fof2**0.5 pred_mufd, sd_mufd = gp.predict(y_mufd, tm / 86400., return_var=True) pred_mufd += mean_mufd sd_mufd = sd_mufd**0.5 pred_hmf2, sd_hmf2 = gp.predict(y_hmf2, tm / 86400., return_var=True) pred_hmf2 += mean_hmf2 sd_hmf2 = sd_hmf2**0.5 for i in range(len(times)): with con.cursor() as cur: cur.execute( """ insert into prediction (run_id, station_id, time, cs, log_stdev, fof2, mufd, hmf2) values (%s, %s, %s, %s, %s, %s, %s, %s) on conflict (station_id, run_id, time) do update set cs=excluded.cs, log_stdev=excluded.log_stdev, fof2=excluded.fof2, mufd=excluded.mufd, hmf2=excluded.hmf2 """, ( run_id, station['id'], times[i], stdev_to_cs(sd_mufd[i]), sd_mufd[i], np.exp(pred_fof2[i]), np.exp(pred_mufd[i]), np.exp(pred_hmf2[i]), )) con.commit() return make_response("OK")
def make_dataset(station): url = 'http://localhost:5502/mixscale.json?station=%d&points=%d&max_span=120' % ( station, points_per_station) data = get_data(url) s = data[0] x, y_fof2, y_mufd, y_hmf2, sigma, cslist = [], [], [], [], [], [] last_tm = None iri = subprocess.Popen( ['/build/iri_opt'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True, ) for pt in s['history']: tm = (pd.to_datetime(pt[0]) - pd.Timestamp("1970-01-01")) // pd.Timedelta("1s") cs = pt[1] iritime = pd.to_datetime(pt[0]).strftime('%Y %m %d %H %M %S') if cs < 10 and cs != -1: continue sd = cs_to_stdev(cs, adj100=True) fof2, mufd, hmf2 = pt[2:5] iri.stdin.write("%f %f %f %s\n" % (s['latitude'], s['longitude'], -100.0, iritime)) iri.stdin.flush() iridata = [float(x) for x in iri.stdout.readline().split()] iri_fof2, iri_mufd, iri_hmf2 = iridata[3], iridata[5], iridata[6] x.append(tm / 86400.) y_fof2.append(np.log(fof2) - np.log(iri_fof2)) y_mufd.append(np.log(mufd) - np.log(iri_mufd)) y_hmf2.append(np.log(hmf2) - np.log(iri_hmf2)) sigma.append(sd) cslist.append(cs) last_tm = tm iri.stdin.close() iri.wait() ds = {} ds['station_id'] = s['id'] ds['latitude'] = s['latitude'] ds['longitude'] = s['longitude'] ds['x'] = np.array(x) ds['last_tm'] = last_tm ds['y_fof2'] = np.array(y_fof2) ds['mean_fof2'] = np.mean(ds['y_fof2']) ds['y_fof2'] -= ds['mean_fof2'] ds['y_mufd'] = np.array(y_mufd) ds['mean_mufd'] = np.mean(ds['y_mufd']) ds['y_mufd'] -= ds['mean_mufd'] ds['y_hmf2'] = np.array(y_hmf2) ds['mean_hmf2'] = np.mean(ds['y_hmf2']) ds['y_hmf2'] -= ds['mean_hmf2'] ds['sigma'] = np.array(sigma) ds['cs'] = np.array(cslist) ds['gp'] = george.GP(delta_kernel) ds['gp'].compute(ds['x'], ds['sigma'] + 1e-3) return ds
def get_datasets(): data = get_data('http://localhost:%s/pred_sample.json?samples=%d' % (os.getenv('API_PORT'), NSAMPLES)) ds = None dss = [] for row in data: if ds is not None and row['run_id'] != ds['run_id']: dss.append(ds) ds = None if ds is None: ds = { 'run_id': row['run_id'], 'measurements': [] } station = stations.get(row['station_name']) if station is None: continue x, y, z = sph_to_xyz(station['latitude'], station['longitude']) meas = { 'lat': station['latitude'], 'lon': station['longitude'], 'x': x, 'y': y, 'z': z, 'fof2': row['fof2'], 'hmf2': row['hmf2'], 'mufd': row['mufd'], 'cs': row['cs'], 'sigma': cs_to_stdev(row['cs']), } ds['measurements'].append(meas) if ds is not None and len(ds['measurements']) > 0: dss.append(ds) for ds in dss: x = np.array([ meas['x'] for meas in ds['measurements'] ]) y = np.array([ meas['y'] for meas in ds['measurements'] ]) z = np.array([ meas['z'] for meas in ds['measurements'] ]) cs = np.array([ meas['cs'] for meas in ds['measurements'] ]) sigma = np.array([ meas['sigma'] for meas in ds['measurements'] ]) fof2 = np.array([ np.log(meas['fof2']) for meas in ds['measurements'] ]) mean_fof2 = np.mean(fof2) fof2 -= mean_fof2 hmf2 = np.array([ np.log(meas['hmf2']) for meas in ds['measurements'] ]) mean_hmf2 = np.mean(hmf2) hmf2 -= mean_hmf2 mufd = np.array([ np.log(meas['mufd']) for meas in ds['measurements'] ]) mean_mufd = np.mean(mufd) mufd -= mean_mufd ds['arrays'] = { 'xyz': np.column_stack((x,y,z)), 'cs': cs, 'sigma': sigma, 'fof2': fof2, 'mean_fof2': mean_fof2, 'hmf2': hmf2, 'mean_hmf2': mean_hmf2, 'mufd': mufd, 'mean_mufd': mean_mufd, } ds['gp'] = george.GP(kernel) ds['gp'].compute(ds['arrays']['xyz'], ds['arrays']['sigma'] + 1e-3) return dss
def generate(): dsn = "dbname='%s' user='******' host='%s' password='******'" % ( os.getenv("DB_NAME"), os.getenv("DB_USER"), os.getenv("DB_HOST"), os.getenv("DB_PASSWORD")) con = psycopg2.connect(dsn) times = [ dt.datetime.fromtimestamp(float(ts)) for ts in request.form.getlist('target') ] if len(times) == 0: times = [dt.datetime.utcnow()] run_id = int(request.form.get('run_id', -1)) data = get_data('http://localhost:%s/history.json?days=14' % (os.getenv('HISTORY_PORT'))) with con.cursor() as cur: cur.execute( 'select ssn from essn where run_id=%s and series=%s order by time desc nulls last limit 1', (run_id, '24h')) (ssn, ) = cur.fetchone() out = [] for station in data: x, y_fof2, y_mufd, y_hmf2, sigma = [], [], [], [], [] for pt in station['history']: ts = pd.to_datetime(pt[0]) tm = (ts - pd.Timestamp("1970-01-01")) // pd.Timedelta("1s") cs = pt[1] if cs < 10 and cs != -1: continue sd = cs_to_stdev(cs, adj100=True) fof2, mufd, hmf2 = pt[2:5] iridata = get_iri(station['latitude'], station['longitude'], ssn, ts) iri_fof2, iri_mufd, iri_hmf2 = iridata[3], iridata[5], iridata[6] x.append(tm / 86400.) y_fof2.append(np.log(fof2) - np.log(iri_fof2)) y_mufd.append(np.log(mufd) - np.log(iri_mufd)) y_hmf2.append(np.log(hmf2) - np.log(iri_hmf2)) sigma.append(sd) if len(x) < 7: continue x = np.array(x) y_fof2 = np.array(y_fof2) mean_fof2 = np.mean(y_fof2) y_fof2 -= mean_fof2 y_mufd = np.array(y_mufd) mean_mufd = np.mean(y_mufd) y_mufd -= mean_mufd y_hmf2 = np.array(y_hmf2) mean_hmf2 = np.mean(y_hmf2) y_hmf2 -= mean_hmf2 sigma = np.array(sigma) gp = george.GP(delta_kernel) gp.compute(x, sigma + 1e-3) tm = np.array([time.mktime(ts.timetuple()) for ts in times]) pred_fof2, sd_fof2 = gp.predict(y_fof2, tm / 86400., return_var=True) pred_fof2 += mean_fof2 sd_fof2 = sd_fof2**0.5 pred_mufd, sd_mufd = gp.predict(y_mufd, tm / 86400., return_var=True) pred_mufd += mean_mufd sd_mufd = sd_mufd**0.5 pred_hmf2, sd_hmf2 = gp.predict(y_hmf2, tm / 86400., return_var=True) pred_hmf2 += mean_hmf2 sd_hmf2 = sd_hmf2**0.5 for i in range(len(times)): iridata = get_iri(station['latitude'], station['longitude'], ssn, times[i]) iri_fof2, iri_mufd, iri_hmf2 = iridata[3], iridata[5], iridata[6] fof2 = np.exp(np.log(iri_fof2) + pred_fof2[i]) mufd = np.exp(np.log(iri_mufd) + pred_mufd[i]) hmf2 = np.exp(np.log(iri_hmf2) + pred_hmf2[i]) delta_fof2 = fof2 - iri_fof2 delta_mufd = mufd - iri_mufd delta_hmf2 = hmf2 - iri_hmf2 with con.cursor() as cur: cur.execute( """ insert into prediction (run_id, station_id, time, cs, log_stdev, fof2, mufd, hmf2) values (%s, %s, %s, %s, %s, %s, %s, %s) on conflict (station_id, run_id, time) do update set cs=excluded.cs, log_stdev=excluded.log_stdev, fof2=excluded.fof2, mufd=excluded.mufd, hmf2=excluded.hmf2 """, ( run_id, station['id'], times[i], stdev_to_cs(sd_mufd[i]), sd_mufd[i], fof2, mufd, hmf2, )) with con.cursor() as cur: cur.execute( """ insert into prediction_delta (run_id, station_id, time, cs, log_stdev, delta_fof2, delta_mufd, delta_hmf2) values (%s, %s, %s, %s, %s, %s, %s, %s) on conflict (station_id, run_id, time) do update set cs=excluded.cs, log_stdev=excluded.log_stdev, delta_fof2=excluded.delta_fof2, delta_mufd=excluded.delta_mufd, delta_hmf2=excluded.delta_hmf2 """, ( run_id, station['id'], times[i], stdev_to_cs(sd_mufd[i]), sd_mufd[i], delta_fof2, delta_mufd, delta_hmf2, )) con.commit() return make_response("OK")