def disp():
flow_out_data = flowOutData()
for i in range(4):
real_data = pd.read_csv(get_data_path())
data = json.load(open('data_run_no_healed{}.json'.format(int(i))))
infectratio = InfectRatio(1, [[0., 1.]], [True])
touchratio = TouchRatio(1, [[0.0, 0.6]], [True])
touchratiointra = TouchRatio(1, [[0, 1]], [True])
obs = ObservationRatio(1, [[0.0, 0.3]], [True])
dead = DeadRatio(1, [[0., 0.01]], [True])
isoratio = IsolationRatio(1, [[0.0, 0.5]], [True])
dummy = DummyModel(1, [[0, 400]], [True, True])
cure_ratio = InfectRatio(1, [[0., 0.1]], [True])
city = 420000
start_date = date(2020, 1, 11)
# set the time of applying touchratio
simulator = Simulator(city,
infectratio,
touchratio,
obs,
dead,
dummy,
isoratio,
touchratiointra,
cure_ratio, [get_important_date(city)],
unob_flow_num=None,
flow_out_data=flow_out_data,
training_date_end=None)
simulator.set_param(data['x'][str(city)])
duration = len(real_data["date"].unique()) - 1
sim_res, _ = simulator.simulate(str(start_date), duration)
plot1(data['real_confirmed'][str(city)], sim_res['observed'], '',
'no_heal{}.pdf'.format(int(i)), 7)
def run_opt(city, budget, start_date, important_dates, infectratio_range=None,
dummy_range=None, unob_flow_num=None, repeat_time=1, init_samples=None,
training_end_date=None, json_name='data_run.json', seed=3, loss_ord=0.0,
touch_range=None,iso_range=None):
assert infectratio_range is not None and dummy_range is not None
days_predict = 0
# load data
real_data = pd.read_csv(get_data_path(True))
history_real = prepareData(real_data)
flow_out_data = None
# initialize models
infectratio = InfectRatio(1, [infectratio_range], [True])
if touch_range is None:
touchratio = TouchRatio(1, [[0.999, 1.0000]], [True])
else:
touchratio = TouchRatio(1, [touch_range], [True])
touchratiointra = TouchRatio(1, [[0, 1]], [True])
obs = ObservationRatio(1, [[0.0, 0.3]], [True])
dead = DeadRatio(1, [[0., 0.01]], [True])
if iso_range is None:
isoratio = IsolationRatio(1, [[0.03, 0.12]], [True])
else:
isoratio = IsolationRatio(1, [iso_range], [True])
dummy = DummyModel(1, [dummy_range], [True, True])
cure_ratio = InfectRatio(1, [[0., 0.15]], [True])
simulator = Simulator(city, infectratio, touchratio, obs, dead, dummy, isoratio, touchratiointra, cure_ratio, important_dates,
unob_flow_num=unob_flow_num, flow_out_data=flow_out_data, training_date_end=training_end_date)
test_date = datetime.strptime(history_real['date'].max(), '%Y-%m-%d').date() - timedelta(days_predict)
history_real = history_real[history_real['adcode'] == city]
history_real = history_real[history_real['date'] >= str(start_date)]
history_train = history_real[history_real['date'] <= str(test_date)]
x, y = simulator.fit(history_train, budget=budget, server_num=get_core_num(),
repeat=repeat_time, seed=seed, intermediate_freq=10000, init_samples=init_samples,
loss_ord=loss_ord)
print('best_solution: x = ', x, 'y = ', y)
simulator.set_param(x)
run_simulation(x, city, 60, 60, start_date, get_important_date(city), unob_flow_num=unob_flow_num, json_name=json_name)
return x
def run_simulation2(x,
city,
simulate_day1,
simulate_day2,
start_date,
simulate_date,
history_real,
incubation=3,
unob_flow_num=None,
json_name='data_run.json'):
days_predict = 0
infectratio = InfectRatio(1, [[0, 1]], [True])
touchratio = TouchRatio(1, [[0, 1.]], [True])
touchratiointra = TouchRatio(1, [[0, 10]], [True])
obs = ObservationRatio(1, [[0.0, 1.]], [True])
dead = DeadRatio(1, [[0., 0.1]], [True])
isoratio = IsolationRatio(1, [[0., 1]], [True])
dummy = DummyModel(1, [[0, 2000000]], [True, True])
cure_ratio = InfectRatio(1, [[0., 100]], [True])
flow_out_data = None
important_dates = [get_important_date(city)]
json_path = json_name
end_date = datetime.strptime(history_real['date'].max(), '%Y-%m-%d').date()
test_date = datetime.strptime(history_real['date'].max(),
'%Y-%m-%d').date() - timedelta(days_predict)
history_real = history_real[history_real['adcode'] == city]
history_real = history_real[history_real['date'] >= str(start_date)]
history_train = history_real[history_real['date'] <= str(test_date)]
duration = len(history_train["date"].unique()) - 1
simulator = Simulator(city,
infectratio,
touchratio,
obs,
dead,
dummy,
isoratio,
touchratiointra,
cure_ratio,
important_dates,
unob_flow_num=unob_flow_num,
flow_out_data=flow_out_data,
incubation=incubation)
if not os.path.exists('./img/{}'.format(city)):
os.makedirs('./img/{}'.format(city))
if not os.path.exists('./result'.format(city)):
os.makedirs('./result'.format(city))
with open('./result/result{}.txt'.format(city), 'w') as f:
print('infection_ratio: [0, 1], touch_ratio_in_province: [0, 0.5]',
file=f)
print('touch_ratio_between_province: [0, 1]', file=f)
print('observe_ratio_each_day: [0.05613, 1]', file=f)
print('dead_ratio_observe: [0, 0.1], dead_ratio_unob: [0, 0.1]',
file=f)
print('initial_infection_hubei: [8, 8.01].', file=f)
print('isolation_ratio: [0, 0.1]', file=f)
print('BEST RESUTL:', file=f)
print('x = ', x, file=f)
simulator.set_param(x)
print('##################################', file=f)
print('HISTORY REAL', file=f)
print('Data form ' + str(start_date) + ' to ' + str(end_date) +
'are used for training',
file=f)
print(history_train[history_train['adcode'] == city], file=f)
print('##################################', file=f)
print('SIMULATED RESULTS', file=f)
print('start_date: ', start_date, 'duration: ', duration, file=f)
sim, records = simulator.simulate(str(start_date), duration)
append_to_json(json_path, 'x', str(city), list(x))
append_to_json(json_path, 'R01', str(city), get_R0(15, x[0], x[5], 14))
append_to_json(json_path, 'R02', str(city),
get_R0(15 * x[1], x[0], x[5], 14))
append_to_json(json_path, 'DT1', str(city), get_DT(15, x[0], x[5]))
append_to_json(json_path, 'DT2', str(city),
get_DT(15 * x[1], x[0], x[5]))
append_to_json(json_path, 'death_rate', str(city),
get_death_rate(x[3], x[7], 14, 10))
append_to_json(json_path, 'real_confirmed', str(city),
list(history_train['observed']))
append_to_json(json_path, 'real_cum_confirmed', str(city),
list(history_train['cum_confirmed']))
append_to_json(json_path, 'sim_confirmed', str(city),
list(sim['observed']))
append_to_json(json_path, 'sim_cum_confirmed', str(city),
list(sim['cum_confirmed']))
append_to_json(json_path, 'loss', str(city),
get_loss(sim, history_train))
append_to_json(json_path, 'newly_confirmed_loss', str(city),
get_newly_loss(sim, history_train))
append_to_json(json_path, 'real_cum_dead', str(city),
list(history_train['cum_dead']))
append_to_json(json_path, 'sim_new_infection', str(city),
list(sim['new_infection']))
append_to_json(json_path, 'sim_total_infection', str(city),
list(sim['total_infection']))
append_to_json(json_path, 'sim_cum_self_cured', str(city),
list(sim['cum_cured']))
append_to_json(json_path, 'sim_cum_nosymbol', str(city),
list(sim['cum_no_symbol']))
print(f'loss: {get_loss(sim, history_train)}', file=f)
cum_real = history_train['cum_confirmed'].values
simulated_data, _ = simulator.simulate(str(start_date), duration)
cum_simulated = simulated_data['cum_confirmed'].values
print(simulated_data[simulated_data['adcode'] == city], file=f)
s1, _ = simulator.simulate(str(start_date),
duration + simulate_day1 + 30)
touch_ratio_high = 0.35 * x[1]
touch_ratio_very_high = 0.6 * x[1]
touch_ratio_low = 0.1 * x[1]
s2, _ = simulator.simulate(str(start_date),
duration + simulate_day2 + 30,
simulate_date=simulate_date,
simulate_touchratio=touch_ratio_high)
s3, _ = simulator.simulate(str(start_date),
duration + simulate_day2 + 30,
simulate_date=simulate_date,
simulate_touchratio=touch_ratio_low)
s8, _ = simulator.simulate(str(start_date),
duration + simulate_day2 + 30,
simulate_date=simulate_date,
simulate_touchratio=touch_ratio_very_high)
print(len(s1['cum_confirmed']), len(s3['cum_confirmed']),
str(start_date + timedelta(len(s3['cum_confirmed']))))
s4, _ = simulator.simulate(str(start_date),
duration + simulate_day2 + 30,
simulate_date=get_important_date(city) +
timedelta(15),
simulate_touchratio=touch_ratio_low)
s5, _ = simulator.simulate(str(start_date),
duration + simulate_day2 + 30,
simulate_date=get_important_date(city) +
timedelta(15),
simulate_touchratio=touch_ratio_high)
s6, _ = simulator.simulate(str(start_date),
duration + simulate_day2 + 30,
simulate_date=get_important_date(city) +
timedelta(5),
simulate_touchratio=touch_ratio_low)
s7, _ = simulator.simulate(str(start_date),
duration + simulate_day2 + 30,
simulate_date=get_important_date(city) +
timedelta(5),
simulate_touchratio=touch_ratio_high)
c_con = list(
s1['cum_confirmed'])[len(list(history_train['observed'])) - 1]
c_inf = list(
s1['cum_infection'])[len(list(history_train['observed'])) - 1]
append_to_json(json_path, 'current_asym', str(city), c_con / c_inf)
c_con = list(s1['cum_confirmed'])[-1]
c_inf = list(s1['cum_infection'])[-1]
append_to_json(json_path, 'final_asym', str(city), c_con / c_inf)
append_to_json(json_path, 'sim_cum_confirmed_deduction_s1', str(city),
list(s1['cum_confirmed']))
append_to_json(json_path, 'sim_cum_confirmed_deduction_s2', str(city),
list(s2['cum_confirmed']))
append_to_json(json_path, 'sim_cum_confirmed_deduction_s3', str(city),
list(s3['cum_confirmed']))
append_to_json(json_path, 'sim_cum_confirmed_deduction_s4', str(city),
list(s4['cum_confirmed']))
append_to_json(json_path, 'sim_cum_confirmed_deduction_s5', str(city),
list(s5['cum_confirmed']))
append_to_json(json_path, 'sim_cum_confirmed_deduction_s6', str(city),
list(s6['cum_confirmed']))
append_to_json(json_path, 'sim_cum_confirmed_deduction_s7', str(city),
list(s7['cum_confirmed']))
append_to_json(json_path, 'sim_cum_confirmed_deduction_s8', str(city),
list(s7['cum_confirmed']))
append_to_json(json_path, 'sim_confirmed_deduction_s1', str(city),
list(s1['observed']))
append_to_json(json_path, 'sim_confirmed_deduction_s2', str(city),
list(s2['observed']))
append_to_json(json_path, 'sim_confirmed_deduction_s3', str(city),
list(s3['observed']))
append_to_json(json_path, 'sim_confirmed_deduction_s4', str(city),
list(s4['observed']))
append_to_json(json_path, 'sim_confirmed_deduction_s5', str(city),
list(s5['observed']))
append_to_json(json_path, 'sim_confirmed_deduction_s6', str(city),
list(s6['observed']))
append_to_json(json_path, 'sim_confirmed_deduction_s7', str(city),
list(s7['observed']))
append_to_json(json_path, 'sim_confirmed_deduction_s8', str(city),
list(s7['observed']))
append_to_json(json_path, 'sim_infection_deduction_s1', str(city),
list(s1['total_infection']))
append_to_json(json_path, 'sim_infection_deduction_s2', str(city),
list(s2['total_infection']))
append_to_json(json_path, 'sim_infection_deduction_s3', str(city),
list(s3['total_infection']))
append_to_json(json_path, 'sim_infection_deduction_s4', str(city),
list(s4['total_infection']))
append_to_json(json_path, 'sim_infection_deduction_s5', str(city),
list(s5['total_infection']))
append_to_json(json_path, 'sim_infection_deduction_s6', str(city),
list(s6['total_infection']))
append_to_json(json_path, 'sim_infection_deduction_s7', str(city),
list(s7['total_infection']))
append_to_json(json_path, 'sim_infection_deduction_s8', str(city),
list(s7['total_infection']))
append_to_json(json_path, 'sim_cum_infection_deduction_s1', str(city),
list(s1['cum_infection']))
append_to_json(json_path, 'sim_cum_infection_deduction_s2', str(city),
list(s2['cum_infection']))
append_to_json(json_path, 'sim_cum_infection_deduction_s3', str(city),
list(s3['cum_infection']))
append_to_json(json_path, 'sim_cum_infection_deduction_s4', str(city),
list(s4['cum_infection']))
append_to_json(json_path, 'sim_cum_infection_deduction_s5', str(city),
list(s5['cum_infection']))
append_to_json(json_path, 'sim_cum_infection_deduction_s6', str(city),
list(s6['cum_infection']))
append_to_json(json_path, 'sim_cum_infection_deduction_s7', str(city),
list(s7['cum_infection']))
append_to_json(json_path, 'sim_cum_infection_deduction_s8', str(city),
list(s7['cum_infection']))
append_to_json(json_path, 'touch_ratio_low', str(city),
touch_ratio_low)
append_to_json(json_path, 'touch_ratio_hight', str(city),
touch_ratio_high)
append_to_json(json_path, 'sim_cum_dead_s1', str(city),
list(s1['cum_dead']))
append_to_json(json_path, 'sim_cum_dead_s2', str(city),
list(s2['cum_dead']))
append_to_json(json_path, 'sim_cum_dead_s3', str(city),
list(s3['cum_dead']))
append_to_json(json_path, 'sim_cum_dead_s4', str(city),
list(s4['cum_dead']))
append_to_json(json_path, 'sim_cum_dead_s5', str(city),
list(s5['cum_dead']))
append_to_json(json_path, 'sim_cum_dead_s6', str(city),
list(s6['cum_dead']))
append_to_json(json_path, 'sim_cum_dead_s7', str(city),
list(s7['cum_dead']))
append_to_json(json_path, 'sim_cum_dead_s8', str(city),
list(s7['cum_dead']))
append_to_json(json_path, 'sim_total_infection_deduction_s1',
str(city), list(s1['total_unobserved']))
append_to_json(json_path, 'sim_total_infection_deduction_s2',
str(city), list(s2['total_unobserved']))
append_to_json(json_path, 'sim_total_infection_deduction_s3',
str(city), list(s3['total_unobserved']))
append_to_json(json_path, 'sim_total_infection_deduction_s4',
str(city), list(s4['total_unobserved']))
append_to_json(json_path, 'sim_total_infection_deduction_s5',
str(city), list(s5['total_unobserved']))
append_to_json(json_path, 'sim_total_infection_deduction_s6',
str(city), list(s6['total_unobserved']))
append_to_json(json_path, 'sim_total_infection_deduction_s7',
str(city), list(s7['total_unobserved']))
append_to_json(json_path, 'sim_total_infection_deduction_s8',
str(city), list(s8['total_unobserved']))
append_to_json(json_path, 'sim_cum_self_cured_deduction_s1', str(city),
list(s1['cum_self_cured']))
append_to_json(json_path, 'sim_cum_self_cured_deduction_s2', str(city),
list(s2['cum_self_cured']))
append_to_json(json_path, 'sim_cum_self_cured_deduction_s3', str(city),
list(s3['cum_self_cured']))
append_to_json(json_path, 'sim_cum_self_cured_deduction_s4', str(city),
list(s4['cum_self_cured']))
append_to_json(json_path, 'sim_cum_self_cured_deduction_s5', str(city),
list(s5['cum_self_cured']))
append_to_json(json_path, 'sim_cum_self_cured_deduction_s6', str(city),
list(s6['cum_self_cured']))
append_to_json(json_path, 'sim_cum_self_cured_deduction_s7', str(city),
list(s7['cum_self_cured']))
append_to_json(json_path, 'sim_cum_self_cured_deduction_s8', str(city),
list(s8['cum_self_cured']))
append_to_json(json_path, 'sim_cum_nosymbol_deduction_s1', str(city),
list(s1['cum_no_symbol']))
append_to_json(json_path, 'sim_cum_nosymbol_deduction_s2', str(city),
list(s2['cum_no_symbol']))
append_to_json(json_path, 'sim_cum_nosymbol_deduction_s3', str(city),
list(s3['cum_no_symbol']))
append_to_json(json_path, 'sim_cum_nosymbol_deduction_s4', str(city),
list(s4['cum_no_symbol']))
append_to_json(json_path, 'sim_cum_nosymbol_deduction_s5', str(city),
list(s5['cum_no_symbol']))
append_to_json(json_path, 'sim_cum_nosymbol_deduction_s6', str(city),
list(s6['cum_no_symbol']))
append_to_json(json_path, 'sim_cum_nosymbol_deduction_s7', str(city),
list(s7['cum_no_symbol']))
append_to_json(json_path, 'sim_cum_nosymbol_deduction_s8', str(city),
list(s8['cum_no_symbol']))
append_to_json(json_path, 'sim_total_isolation_deduction_s1',
str(city), list(s1['total_isolation']))
append_to_json(json_path, 'sim_total_isolation_deduction_s2',
str(city), list(s2['total_isolation']))
append_to_json(json_path, 'sim_total_isolation_deduction_s3',
str(city), list(s3['total_isolation']))
append_to_json(json_path, 'sim_total_isolation_deduction_s4',
str(city), list(s4['total_isolation']))
append_to_json(json_path, 'sim_total_isolation_deduction_s5',
str(city), list(s5['total_isolation']))
append_to_json(json_path, 'sim_total_isolation_deduction_s6',
str(city), list(s6['total_isolation']))
append_to_json(json_path, 'sim_total_isolation_deduction_s7',
str(city), list(s7['total_isolation']))
append_to_json(json_path, 'sim_total_isolation_deduction_s8',
str(city), list(s8['total_isolation']))
def run_opt(city,
budget,
start_date,
important_dates,
infectratio_range=None,
dummy_range=None,
unob_flow_num=None,
repeat_time=1,
init_samples=None,
training_date_end=None,
json_name='data_run.json',
seed=1,
loss_ord=0.0,
unob_period=None,
obs_period=None,
iso_period=None,
cure_period=None,
isoratio_it=None):
if city == 420000:
infectratio_range = [0., 0.05]
dummy_range = [0.0000, 400.00001]
else:
assert infectratio_range is not None and dummy_range is not None
days_predict = 0
# load data
real_data = pd.read_csv(get_data_path())
history_real = prepareData(real_data)
flow_out_data = flowOutData()
# initialize models
infectratio = InfectRatio(1, [infectratio_range], [True])
touchratio = TouchRatio(1, [[0.0, 0.6]], [True])
touchratiointra = TouchRatio(1, [[0, 1]], [True])
obs = ObservationRatio(1, [[0.0, 0.3]], [True])
dead = DeadRatio(1, [[0., 0.01]], [True])
if isoratio_it is None:
isoratio = IsolationRatio(1, [[0.2, 0.5]], [True])
else:
isoratio = IsolationRatio(1, [isoratio_it], [True])
dummy = DummyModel(1, [dummy_range], [True, True])
cure_ratio = InfectRatio(1, [[0., 0.1]], [True])
# set the time of applying touchratio
simulator = Simulator(city,
infectratio,
touchratio,
obs,
dead,
dummy,
isoratio,
touchratiointra,
cure_ratio,
important_dates,
unob_flow_num=unob_flow_num,
flow_out_data=flow_out_data,
training_date_end=training_date_end)
# set period here
simulator.set_period()
test_date = datetime.strptime(history_real['date'].max(),
'%Y-%m-%d').date() - timedelta(days_predict)
history_real = history_real[history_real['adcode'] == city]
history_real = history_real[history_real['date'] >= str(start_date)]
history_train = history_real[history_real['date'] <= str(test_date)]
x, y = simulator.fit(history_train,
budget=budget,
server_num=get_core_num(),
repeat=repeat_time,
seed=seed,
intermediate_freq=10000,
init_samples=init_samples,
loss_ord=loss_ord)
print('best_solution: x = ', x, 'y = ', y)
simulator.set_param(x)
run_simulation(x,
city,
60,
60,
start_date,
get_important_date(city),
unob_flow_num=unob_flow_num,
json_name=json_name)
duration = len(real_data["date"].unique()) - 1
sim_res, _ = simulator.simulate(str(start_date), duration)
print('RMSE: ', get_newly_loss(sim_res, history_real))
return x, sim_res
def initHubei(x, start_date, important_date, travel_from_hubei):
if travel_from_hubei is None:
return None
days_predict = 0
# load data
real_data = pd.read_csv(get_data_path())
history_real = prepareData(real_data)
infectratio = InfectRatio(1, [[0, 1]], [True])
touchratio = TouchRatio(1, [[0., 0.3]], [True])
touchratiointra = TouchRatio(1, [[0, 10]], [True])
obs = ObservationRatio(1, [[0.0, 1.]], [True])
dead = DeadRatio(1, [[0., 0.1]], [True])
isoratio = IsolationRatio(1, [[0., 1]], [True])
cure_ratio = InfectRatio(1, [[0., 100]], [True])
dummy = DummyModel(1, [[0, 200000]], [True, True])
flow_out_data = flowOutData()
test_date = datetime.strptime(history_real['date'].max(),
'%Y-%m-%d').date() - timedelta(days_predict)
history_real = history_real[history_real['adcode'] == 420000]
history_real = history_real[history_real['date'] >= str(start_date)]
history_train = history_real[history_real['date'] <= str(test_date)]
duration = len(history_train["date"].unique())
city = 420000
simulator = Simulator(city,
infectratio,
touchratio,
obs,
dead,
dummy,
isoratio,
touchratiointra,
cure_ratio,
important_date,
flow_out_data=flow_out_data)
simulator.set_param(x)
total_population = get_populations()[420000]
simulated_result, detailed_result = simulator.simulate(
str(start_date), duration + 60)
init_unob = [item for item in reversed(detailed_result.unob_total_init)]
unob_ratio = {}
for i, item in enumerate(init_unob):
date_now = start_date - timedelta(i + 1)
unob_ratio[date_now] = item / total_population
for it_date, unob_num in zip(simulated_result['date'],
simulated_result['total_unobserved']):
unob_ratio[datetime.strptime(
it_date, '%Y-%m-%d').date()] = unob_num / total_population
unob_flow_num = {}
lst_travel = {}
lst_date = None
for it_date in travel_from_hubei:
if it_date not in unob_ratio:
continue
if it_date not in unob_flow_num:
unob_flow_num[it_date] = {}
for it_code in travel_from_hubei[it_date]:
unob_flow_num[it_date][it_code] = travel_from_hubei[it_date][
it_code] * unob_ratio[it_date]
lst_travel = travel_from_hubei[it_date]
lst_date = it_date
lst_date = lst_date + timedelta(1)
while True:
if lst_date in unob_ratio:
if lst_date not in unob_flow_num:
unob_flow_num[lst_date] = {}
for it_code in lst_travel:
unob_flow_num[lst_date][
it_code] = lst_travel[it_code] * unob_ratio[lst_date]
else:
break
lst_date = lst_date + timedelta(1)
return unob_flow_num