def load_reference(self):
reference_file, _ = self._args.files
with open(reference_file) as fh:
self._reference_partition = json.load(fh)
if has_flag(self._reference_partition, Flag.INDEXED):
pass
else:
self._reference_partition = indexed_partition(self._reference_partition)
def convert2idx(fname):
fidx = fname.replace('.input.json', '.idx.input.json')
if fidx == fname:
print("Couldn't change file name. Do nothing.")
return
with open(fname) as fh:
data = json.load(fh)
if has_flag(data, Flag.INDEXED):
return
with open(fidx, 'w') as fh:
json.dump(indexed_partition(data), fh)
print('Wrote "%s"' % fidx)
def simulation_posterior(initial_partition, args):
if has_flag(initial_partition, Flag.INDEXED):
pass
else:
initial_partition = indexed_partition(initial_partition)
parameters = init_parameters()
parameters.initQ(args.process_noise*2)
have_sims, input_partition, sims_partition, pool = convert_initial_partition(initial_partition)
if not have_sims:
log.info("NO simulation found.")
return
posts = []
for i in range(10000):
sample_model_parameters_given_partition(sims_partition, parameters)
posts.append(log_pdf(sims_partition, parameters))
log.info("Best PDF = %f" % max(posts))
def track_stepper(initial_partition, args):
'''Does tracking without burn-in control (uses Stepper)
Stops after given time or given number of samples or keyboard interrupt.
An eventual recording follows after the stop signal
'''
out_format = 'json'
if has_flag(initial_partition, Flag.INDEXED):
pass
else:
log.info('convert to indexed partition')
initial_partition = indexed_partition(initial_partition)
total_obs = len(initial_partition['observations'])
if args.max:
maxi_tracks, maxi_clutter = max_partition(initial_partition['observations'])
pdfs_len = 200
current_pdfs = fifo(pdfs_len)
if not args.norecord:
'''open the file asap'''
directory = get_tracking_dir(args.input)
output_file = open(get_output_filename(args.input, directory,
"tracked." + initial_partition['rand_id']), 'w')
recorder = SampleRecorder(initial_partition)
initial_model = init_parameters()
initial_model.initQ(args.process_noise*2)
have_sims, input_partition, sims_partition, pool = convert_initial_partition(initial_partition)
singletons = make_mini_partition(initial_partition['observations'], pool)
if have_sims:
ged_list = fifo(200)
if not args.norecord:
recorder.set_sims_partition(sims_partition)
# Create a new tracking server
dancer = None
if args.max:
maxi_partition = IndexedPartition()
maxi_partition.set_tracks(maxi_tracks)
maxi_partition.set_clutter(maxi_clutter)
dancer = Stepper(initial_model, maxi_partition.to_partition(pool))
else:
#dancer = Stepper(initial_model, input_partition)
dancer = Stepper(initial_model, singletons)
print_r(dancer.tracking_state.current_model_parameters, log.info)
state_file = None
if args.log_states is not None:
state_file = safe_open(args.log_states, 'w', args.unsafe, log.error)
log.info('Writing states one-per-line to: %s' % (args.log_states,))
start_time = time.time()
#tick = 0
#tick_header_interval = 10
#dancer.step(1000)
state = dancer.tracking_state
stored_log_pdf = log_pdf(state.current_partition, initial_model)
last_best_partition = state.current_partition
header_counter = 0
header_lag = 15
seconds_passed = 0
one_hour = 3600
count_chunk = 100000 # for reporting
samples_passed = 0 # for reporting
try:
while True:
dancer.step(1000)
state = dancer.tracking_state
current_pdfs.append(state.current_log_pdf)
tick_time = time.time() - start_time
if have_sims:
ged_list.append(state.current_partition.ged(sims_partition))
if state.best_log_pdf > stored_log_pdf or (tick_time - seconds_passed) > one_hour or \
(state.sample_count-samples_passed)>count_chunk:
while (tick_time - seconds_passed) > one_hour:
seconds_passed += one_hour
while (state.sample_count-samples_passed)>count_chunk:
samples_passed += count_chunk
if header_counter % header_lag == 0:
print_header(total_obs, state, have_sims)
header_counter = (header_counter + 1) % header_lag
hist = list(state.move_histogram)
accepted = float(sum(hist[1:-1]))
total = float(sum(hist))
assert(total == state.sample_count)
total /= 100
improvement = 100.0*(stored_log_pdf - state.best_log_pdf)/stored_log_pdf
ged = last_best_partition.ged(state.best_partition)
if have_sims:
ged = min(ged_list)
#ged = state.best_partition.ged(sims_partition)
ged_str = four2str(ged)
last_best_partition = state.best_partition
log.info('%12s %7s %8s %6.2f %6.2f %6.2f %8.1f %6.1f%% %4d %6.2f%% %4s' % (
sec2str(int(tick_time)), count2str(state.sample_count),
float2str(state.best_log_pdf),
float(hist[0]) / total, float(hist[-1])/total, accepted/total,
state.sample_count/float(tick_time),
100.0*float(len(state.best_partition.clutter))/float(total_obs),
len(state.best_partition.tracks),
improvement, ged_str))
stored_log_pdf = state.best_log_pdf
# Log state if required
if state_file is not None:
# Add time field to state
state_d = {
'sample_count': state.sample_count,
'log_pdf': state.current_log_pdf,
'partition': json.loads(state.current_partition.to_json()),
'model_parameters': json.loads(state.current_model_parameters.to_json()),
'move_type': int(state.current_move_type),
'acceptance_rate': state.acceptance_rate,
'move_histogram': list(state.move_histogram),
}
json.dump(state_d, state_file)
state_file.write('\n')
if args.duration and args.duration < tick_time:
log.info('Stopping since duration has exceeded %s seconds.' % (args.duration,))
break
if args.samples and number_parser(args.samples, int) < state.sample_count:
log.info('Stopping since sample count has exceeded %s.' % (args.samples,))
break
if args.mean_pdf and len(current_pdfs) == pdfs_len and mean(current_pdfs) > -args.mean_pdf:
log.info('Stopping since mean log PDF exceeded %.1f' % (-args.mean_pdf,))
break
#tick+=1
except KeyboardInterrupt:
log.warning('Terminating tracking due to keyboard interrupt.')
return 1
finally:
state = dancer.tracking_state
# del dancer
if not args.norecord:
recorder.set_burnin_length(state.sample_count)
record_samples(state, recorder, args.records)
if out_format == 'json':
recorder.write_results(args, output_file)
if out_format == 'hdf5':
hdf5name = get_output_filename(args.input, 'tracking')
with h5py.File(hdf5name.replace('.json', '.hdf5')) as fh:
recorder.write_hdf5_results(args, fh)
print_final_state(log.info, state)
if args.log_best:
fname = get_output_filename(
args.input, '.', 'best.' + initial_partition['rand_id'], insert_time = True)
if out_format == 'json':
with open(fname, 'w') as fh:
#write_input_partition(state.best_partition, args, fh)
write_partition_indices(state.best_partition, pool, args, fh)
if out_format == 'hdf5':
index_part = IndexedPartition()
index_part.from_partition(state.best_partition)
with h5py.File(fname.replace('.json', '.hdf5')) as fh:
write_hdf5_observations([ (o.x, o.y, o.t) for o in pool] ,fh)
write_hdf5_indexed_partition(index_part, fh)
log.info("Run time %s" % sec2str(time.time()-start_time))
## Stop the tracker.
#tracker.stop()
return 0
def track_dual(initial_partition, args):
'''does tracking with dual burn-in'''
out_format = 'json'
if has_flag(initial_partition, Flag.INDEXED):
pass
else:
initial_partition = indexed_partition(initial_partition)
if not args.norecord:
directory = get_tracking_dir(args.input)
output_file = open(
get_output_filename(args.input, directory,
"tracked." + initial_partition['rand_id']), 'w')
recorder = SampleRecorder(initial_partition)
initial_model = init_parameters()
initial_model.initQ(args.process_noise*2)
have_sims, input_partition, sims_partition, pool = convert_initial_partition(initial_partition)
dancers = []
start_partitions = []
split_part = None
singletons = make_mini_partition(initial_partition['observations'], pool)
for part_type in args.start:
if part_type == "input":
dancers.append(Tracker(Stepper(initial_model, input_partition)))
start_partitions.append(input_partition)
elif part_type == "maxi":
start_partition = make_maxi_partition(initial_partition['observations'], pool)
dancers.append(Tracker(Stepper(initial_model, start_partition)))
start_partitions.append(start_partition)
elif part_type == "rev":
start_partition = make_rev_partition(initial_partition['observations'], pool)
dancers.append(Tracker(Stepper(initial_model, start_partition)))
start_partitions.append(start_partition)
elif part_type == "mini":
dancers.append(Tracker(Stepper(initial_model, singletons)))
start_partitions.append(singletons)
elif part_type == "split":
if split_part is None:
half1, split_part = make_split_partition(initial_partition['observations'], pool)
dancers.append(Tracker(Stepper(initial_model, half1)))
start_partitions.append(half1)
else:
dancers.append(Tracker(Stepper(initial_model, split_part)))
start_partitions.append(split_part)
start_time = time.time()
min_sample_count = args.min_samples
for dancer in dancers:
dancer.sample_observation_error(args.obs_cov_lag)
log.info('convergence method = %s' % (args.convergence,))
ctrl = ControlData(start_time, len(pool), args.convergence, args.fifo_length)
ctrl.print_explaination(log.info)
ctrl.print_header_explain(log.info)
ctrl.set_logger(log.info)
ctrl.report_ac_time(args.report_act)
ctrl.set_start_partitions(start_partitions)
if args.log_all:
fname = get_output_filename(args.input, '.', 'all.' + initial_partition['rand_id'])
fname = fname.replace('.json', '.csv')
ctrl.write_all_log(fname)
max_ged = get_max_ged(initial_partition['observations'])
ctrl.set_max_ged(max_ged)
if have_sims:
log.info("simulation found.")
ctrl.set_simulation(sims_partition)
timer = TickTimer(lag = 2)
loop_size = 25
do_record = False
init_ged = dancers[0].state().current_partition.ged(dancers[1].state().current_partition)
log.info("Max GED = {1}, Initial distance = {0:5.1f}%".format(
100.0*init_ged/float(max_ged), max_ged))
try:
for i in range(ctrl.buffer_size()):
ctrl.states = [dancer.run(1) for dancer in dancers]
ctrl.set_sampling_rates([dancer.sample_rate for dancer in dancers])
while True:
ctrl.states = [dancer.run(loop_size) for dancer in dancers]
ctrl.set_sampling_rates([dancer.sample_rate for dancer in dancers])
tick_time = time.time() - start_time
if timer.passed(tick_time):
ctrl.try_output(tick_time)
if ctrl.converged() and ctrl.sample_count >= min_sample_count:
end_time = time.time() - start_time
log.info('Tracker has converged after %s' % sec2str(end_time))
do_record = not args.norecord
if args.run_time:
fh = open(get_output_filename(args.input, '.', 'time', insert_time = False), 'a')
fh.write("{0:.1f}\n".format(end_time))
break
if args.duration and args.duration < tick_time:
log.info('Stopping since duration has exceeded %s seconds.' % (args.duration,))
break
if args.samples and number_parser(args.samples, int) <= ctrl.sample_count:
log.info('Stopping since sample count has exceeded %s.' % (args.samples,))
break
if not do_record:
return 1
dancer = dancers[0]
if (dancers[1].state().best_log_pdf > dancers[0].state().best_log_pdf):
dancer = dancers[1]
recorder.set_burnin_length(dancer.state().sample_count)
dancer.sample_observation_error(1)
# this is not necessary
# dancer.run(5000) # do some sampling with new 'sample_observation_error' setting
num_partiton_links = list()
fact = 8.0
num_steps = max(1, int(fact*0.25/dancer.state().recent_acceptance_rate))
log.info("Recent acceptance rate = %.2f%%" % (100.0*dancer.state().recent_acceptance_rate))
log.info("Recording every %d sample (factor = %.2f)" % (num_steps, fact))
accepted = 0.
last_partition = dancer.state().current_partition
for i in range(args.records):
recorder.record_state(dancer.run(num_steps))
if last_partition.ged(dancer.state().current_partition) > 0:
accepted += 1.
num_partiton_links.append(singletons.ged(dancer.state().current_partition))
last_partition = dancer.state().current_partition
#if have_sims:
# pass
recorder.write_results(args, output_file)
log.info("ACT number of links: %.2f" % auto_corr_time(num_partiton_links))
print_record_act(recorder, log.info)
log.info("Partition change rate during recording: %.2f%%" % (100.*accepted/args.records))
except KeyboardInterrupt:
log.warning('Terminating tracking due to keyboard interrupt.')
return 1
except SignalReceived as sig:
for line in traceback.format_stack():
log.warning(line.strip())
log.warning('Terminating tracking due to user signal {0}'.format(sig))
return 1
#except Exception as ex:
#for line in traceback.format_stack():
# log.warning(line.strip())
#log.warning('Terminating tracking due to exception "{0}"'.format(ex))
#raise ex
finally:
ctrl.print_criterion()
for chain, dancer in enumerate(dancers):
log.info("Chain-%d:" % chain)
print_final_state(log.info, dancer.stepper.tracking_state)
log.info("Run time %s" % sec2str(time.time()-start_time))
if args.log_ged:
fname = get_output_filename(args.input, '.', 'ged_time', insert_time = True)
ctrl.write_ged_over_time(open(fname.replace('.json', '.csv'), 'w'))
if args.log_best:
for chain, dancer in enumerate(dancers):
fname = get_output_filename(args.input, '.', 'best%d.%s' % (
chain, initial_partition['rand_id']), insert_time = True)
if out_format == 'json':
with open(fname, 'w') as fh:
#write_input_partition(dancer.stepper.tracking_state.best_partition, args, fh)
write_partition_indices(
dancer.stepper.tracking_state.best_partition, pool, args, fh)
if out_format == 'hdf5':
index_part = IndexedPartition()
index_part.from_partition(dancer.stepper.tracking_state.best_partition)
with h5py.File(fname.replace('.json', '.hdf5')) as fh:
write_hdf5_observations([ (o.x, o.y, o.t) for o in pool] ,fh)
write_hdf5_indexed_partition(index_part, fh)