def accumulate_frame(handler, acc_dict, frame):
'''
Calculate the correlation products of a data frame and
add it to an accumulator. If the accumulator is full,
then write the data to disk.
'''
if acc_dict['filled'] == 0:
acc_dict['timestamp'] = frame.packet_timestamps[0]
acc_dict['index'] = frame.index
# print frame.index
# for i in xrange(4):
# print frame.packet_timestamps[i]
datum = correlator.convert_frame_multiple_channels(frame.frame_data)
acc_dict['accumulator'] += correlator.return_triangle_array(
correlator.correlate(datum))
acc_dict['frame_indices'][acc_dict['filled']] = frame.index
acc_dict['frame_numbers'][acc_dict['filled']] = frame.frame_number
acc_dict['filled'] += 1
if acc_dict['filled'] == acc_dict['n_acc']:
write_to_disk(handler, acc_dict)
acc_dict['write_index'] += 1
acc_dict['accumulator'] *= 0
acc_dict['frame_indices'] *= 0
acc_dict['frame_numbers'] *= 0
acc_dict['filled'] = 0
def test_correlator(self):
radar = rlib.eiscat_uhf()
measurement_folder = './data/uhf_test_data/events'
tle_file = './data/uhf_test_data/tle-201801.txt'
measurement_file = measurement_folder + '/det-000000.h5'
with h5py.File(measurement_file, 'r') as h_det:
r = h_det['r'].value * 1e3
t = h_det['t'].value
v = -h_det['v'].value
dat = {
'r': r,
't': t,
'v': v,
}
pop = population_library.tle_snapshot(tle_file, sgp4_propagation=True)
pop.filter('oid', lambda oid: n.abs(oid - 43075) < 50)
cdat = correlator.correlate(
data=dat,
station=radar._rx[0],
population=pop,
metric=correlator.residual_distribution_metric,
n_closest=2,
out_file=None,
verbose=False,
MPI_on=False,
)
assert int(cdat[0]['sat_id']) == 43075
self.assertLess(n.abs(cdat[0]['stat'][0]), 1e3)
self.assertLess(n.abs(cdat[0]['stat'][1]), 500)
self.assertLess(n.abs(cdat[0]['stat'][2]), 10.0)
self.assertLess(n.abs(cdat[0]['stat'][3]), 5.0)
def calculate(output_file, input_file_0, input_file_1, threshold):
if threshold > 0:
correlator.correlate(output_file, input_file_0, input_file_1, spearman="spearman", rho_threshold = threshold)
else:
correlator.correlate(output_file, input_file_0, input_file_1, spearman="spearman")
def main():
init()
download()
correlate()
upload()
def test_correlate(self):
results_dir = os.path.abspath("../result")
correlate()
self.assertEqual(len(os.listdir(results_dir)),
2,
msg="Expected correlate to generate two city files")
print('r_obs: ',len(r_obs))
print('r_sim: ',len(r_sim))
'''
dat = {
't': t_sim,
'r': r_sim * 1e3,
'v': v_sim * 1e3,
}
cdat = correlator.correlate(
data=dat,
station=radar._rx[0],
population=pop,
metric=correlator.residual_distribution_metric,
n_closest=1,
out_file=None,
verbose=False,
MPI_on=False,
)
correlator.plot_correlation(dat, cdat[0])
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(211)
ax.plot(t_sim - t_sim[0], r_sim - r_obs, 'ob')
ax.set_ylabel('sim vs obs error [km]')
ax.set_xlabel('relative time [s]')
ax.set_title('Real TDM vs simulated TDM')
#print(v_sim)
def test_create_tracklet(self):
radar = rlib.eiscat_uhf()
radar.set_FOV(30.0, 25.0)
#tle files for envisat in 2016-09-05 to 2016-09-07 from space-track.
TLEs = [
('1 27386U 02009A 16249.14961597 .00000004 00000-0 15306-4 0 9994',
'2 27386 98.2759 299.6736 0001263 83.7600 276.3746 14.37874511760117'
),
('1 27386U 02009A 16249.42796553 .00000002 00000-0 14411-4 0 9997',
'2 27386 98.2759 299.9417 0001256 82.8173 277.3156 14.37874515760157'
),
('1 27386U 02009A 16249.77590267 .00000010 00000-0 17337-4 0 9998',
'2 27386 98.2757 300.2769 0001253 82.2763 277.8558 14.37874611760201'
),
('1 27386U 02009A 16250.12384028 .00000006 00000-0 15974-4 0 9995',
'2 27386 98.2755 300.6121 0001252 82.5872 277.5467 14.37874615760253'
),
('1 27386U 02009A 16250.75012691 .00000017 00000-0 19645-4 0 9999',
'2 27386 98.2753 301.2152 0001254 82.1013 278.0311 14.37874790760345'
),
]
pop = population_library.tle_snapshot(TLEs, sgp4_propagation=True)
#it seems to around 25m^2 area
d = n.sqrt(25.0 * 4 / n.pi)
pop.add_column('d', space_object_uses=True)
pop['d'] = d
ccsds_file = './data/uhf_test_data/events/2002-009A-1473150428.tdm'
obs_data = ccsds_write.read_ccsds(ccsds_file)
jd_obs = dpt.mjd_to_jd(dpt.npdt2mjd(obs_data['date']))
date_obs = obs_data['date']
sort_obs = n.argsort(date_obs)
date_obs = date_obs[sort_obs]
r_obs = obs_data['range'][sort_obs]
jd_sort = jd_obs.argsort()
jd_obs = jd_obs[jd_sort]
jd_det = jd_obs[0]
jd_pop = dpt.mjd_to_jd(pop['mjd0'])
pop_id = n.argmin(n.abs(jd_pop - jd_det))
obj = pop.get_object(pop_id)
print(obj)
jd_obj = dpt.mjd_to_jd(obj.mjd0)
print('Day difference detection - TLE: {}'.format(jd_det - jd_obj))
t_obs = (jd_obs - jd_obj) * (3600.0 * 24.0)
meas, fnames, ecef_stdevs = simulate_tracklet.create_tracklet(
obj,
radar,
t_obs,
hdf5_out=True,
ccsds_out=True,
dname="./tests/tmp_test_data",
noise=False,
)
out_h5 = fnames[0] + '.h5'
out_ccsds = fnames[0] + '.tdm'
print('FILES: ', fnames)
with h5py.File(out_h5, 'r') as h_det:
assert 'm_range' in h_det
assert 'm_range_rate' in h_det
assert 'm_time' in h_det
sim_data = ccsds_write.read_ccsds(out_ccsds)
date_sim = sim_data['date']
sort_sim = n.argsort(date_sim)
date_sim = date_sim[sort_sim]
r_sim = sim_data['range'][sort_sim]
v_sim = sim_data['doppler_instantaneous'][sort_sim]
lt_correction = n.round(r_sim / scipy.constants.c * 1e6).astype(
n.int64).astype('timedelta64[us]')
date_sim_cor = date_sim + lt_correction
t_sim = dpt.jd_to_unix(dpt.mjd_to_jd(dpt.npdt2mjd(date_sim_cor)))
for ind in range(len(date_sim)):
time_df = (dpt.npdt2mjd(date_sim_cor[ind]) -
dpt.npdt2mjd(date_obs[ind])) * 3600.0 * 24.0
assert time_df < 0.01
assert len(r_obs) == len(r_sim)
dat = {
't': t_sim,
'r': r_sim * 1e3,
'v': v_sim * 1e3,
}
cdat = correlator.correlate(
data=dat,
station=radar._rx[0],
population=pop,
metric=correlator.residual_distribution_metric,
n_closest=1,
out_file=None,
verbose=False,
MPI_on=False,
)
self.assertLess(n.abs(cdat[0]['stat'][0]), 5.0)
self.assertLess(n.abs(cdat[0]['stat'][1]), 50.0)
self.assertLess(n.abs(cdat[0]['stat'][2]), 5.0)
self.assertLess(n.abs(cdat[0]['stat'][3]), 50.0)
nt.assert_array_less(n.abs(r_sim - r_obs), 1.0)
os.remove(out_h5)
print('removed "{}"'.format(out_h5))
os.remove(out_ccsds)
print('removed "{}"'.format(out_ccsds))
sat_folder = os.sep.join(fnames[0].split(os.sep)[:-1])
os.rmdir(sat_folder)
print('removed "{}"'.format(sat_folder))
__author__ = 'guorongxu'
import sys
import correlator
if __name__ == "__main__":
row_filename = sys.argv[1]
col_filename = sys.argv[1]
print "The input file name: " + row_filename
outfile = row_filename.replace(".txt", ".cor")
correlator.correlate(outfile,
row_filename,
col_filename,
spearman=True,
pvalue_threshold=None,
rho_threshold=0.3)
