def get_data(config):
return (
loader.read_data(
"br",
config,
endpoint=config["br"]["api"]["endpoints"]["farolcovid"]["cities"],
).replace({"medio": "médio", "insatisfatorio": "insatisfatório"}),
loader.read_data(
"br",
config,
endpoint=config["br"]["api"]["endpoints"]["farolcovid"]["states"],
).replace({"medio": "médio", "insatisfatorio": "insatisfatório"}),
)
def get_data(config):
dfs = {
place: loader.read_data(
"br",
config,
endpoint=config["br"]["api"]["endpoints"]["farolcovid"][place],
).pipe(utils.fix_dates)
for place in ["city", "health_region", "state"]
}
cnes_sources = loader.read_data("br", config, "br/cities/cnes")
# places_ids = loader.read_data("br", config, "br/places/ids")
return dfs, cnes_sources
def _prepare_simulation(user_input, config):
# based on Alison Hill: 40% asymptomatic
user_input["population_params"]["I"] = [
int(user_input["population_params"]["I"] *
(1 - config["br"]["seir_parameters"]["asymptomatic_proportion"]))
if not np.isnan(user_input["population_params"]["I"]) else 1
][0]
place_specific_params = loader.read_data(
"br",
config,
endpoint=config["br"]["api"]["endpoints"]["parameters"][
user_input["place_type"]],
).set_index(user_input["place_type"])
place_id = user_input["place_type"]
user_input["place_specific_params"] = {
"fatality_ratio":
place_specific_params["fatality_ratio"].loc[int(user_input[place_id])],
"i1_percentage":
place_specific_params["i1_percentage"].loc[int(user_input[place_id])],
"i2_percentage":
place_specific_params["i2_percentage"].loc[int(user_input[place_id])],
"i3_percentage":
place_specific_params["i3_percentage"].loc[int(user_input[place_id])],
}
return user_input
def loading_cached_cities(params):
return loader.read_data(
"br",
loader.config,
endpoint=loader.config["br"]["api"]["endpoints"]["analysis"]["cases"],
params=params,
)
def get_rt(place_type, user_input, config, bound):
if place_type == "city_id":
col = place_type
endpoint = config["br"]["api"]["endpoints"]["rt_cities"]
if place_type == "state_id":
col = "state"
endpoint = config["br"]["api"]["endpoints"]["rt_states"]
rt = loader.read_data("br", config, endpoint=endpoint)
# pegando estimacao de 10 dias atras
rt = rt[rt["last_updated"] == (rt["last_updated"].max() -
dt.timedelta(10))]
# caso nao tenha rt, usa o rt do estado
if (place_type == "city_id") & (user_input[place_type]
not in rt[col].values):
return get_rt("state_id", user_input, config, bound)
cols = {"best": "Rt_low_95", "worst": "Rt_high_95"}
if user_input["strategy"] == "isolation": # current
return rt[rt[col] == user_input[place_type]][cols[bound]].values[0]
if user_input["strategy"] == "lockdown": # smaller_rt
return rt[rt[col] == user_input[place_type]][cols[bound]] / 2
if user_input["strategy"] == "nothing": # greater_rt
return rt[rt[col] == user_input[place_type]][cols[bound]] * 2
def get_data(config):
dfs = {
place: loader.read_data(
"br",
config,
endpoint=config["br"]["api"]["endpoints"]["farolcovid"][place],
).replace({
"medio": "médio",
"insatisfatorio": "insatisfatório"
}).pipe(utils.fix_dates)
for place in ["city", "health_region", "state"]
}
places_ids = loader.read_data("br", config, "br/places/ids")
return dfs, places_ids
def check_initialize(self):
if self.dictionary is None:
self.dictionary = loader.read_data(
"br",
loader.config,
loader.config["br"]["api"]["endpoints"]["utilities"]
["place_ids"],
)
def main(session_state=None):
user_analytics = amplitude.gen_user(utils.get_server_session())
opening_response = user_analytics.safe_log_event("opened analysis",
session_state,
is_new_page=True)
utils.localCSS("style.css")
utils.localCSS("icons.css")
config = yaml.load(open("configs/config.yaml", "r"),
Loader=yaml.FullLoader)
br_cases = loader.read_data(
"br",
config,
endpoint=config["br"]["api"]["endpoints"]["analysis"]["cases"])
st.write(
"""
<div class="base-wrapper">
<span class="section-header primary-span">MORTES DIÁRIAS POR MUNICÍPIO</span>
</div>
""",
unsafe_allow_html=True,
)
user_uf = st.selectbox("Selecione um estado para análise:",
utils.get_ufs_list())
prepare_heatmap(
br_cases,
place_type="city_name",
group=user_uf,
)
prepare_heatmap(
br_cases,
place_type="state_id",
)
prepare_heatmap(
loader.read_data(
"br",
config,
endpoint=config["br"]["api"]["endpoints"]["analysis"]["owid"]),
place_type="country_pt",
)
def load_countries_heatmap(config):
st.write("")
da.prepare_heatmap(
loader.read_data(
"br",
loader.config,
endpoint=config["br"]["api"]["endpoints"]["analysis"]["owid"],
),
place_type="country_pt",
)
def read(config):
counter = get_counter(config.train_file)
if os.path.exists(config.emb_dict):
with open(config.emb_dict, "r") as fh:
emb_dict = json.load(fh)
else:
emb_dict = read_glove(config.glove_word_file, counter,
config.glove_word_size, config.glove_dim)
with open(config.emb_dict, "w") as fh:
json.dump(emb_dict, fh)
word2idx_dict, fixed_emb, traiable_emb = token2id(config, counter,
emb_dict)
train_data = read_data(config.train_file)
dev_data = read_data(config.dev_file)
test_data = read_data(config.test_file)
pretrain_data = read_pretrain(config)
pretrain_data2 = read_pretrain(config, 2)
return word2idx_dict, fixed_emb, traiable_emb, train_data, dev_data, test_data, pretrain_data, pretrain_data2
def main():
utils.localCSS("style.css")
utils.localCSS("icons.css")
config = yaml.load(open("configs/config.yaml", "r"),
Loader=yaml.FullLoader)
br_cases = loader.read_data(
"br",
config,
endpoint=config["br"]["api"]["endpoints"]["analysis"]["cases"])
st.write(
"""
<div class="base-wrapper">
<span class="section-header primary-span">MORTES DIÁRIAS POR MUNICÍPIO</span>
</div>
""",
unsafe_allow_html=True,
)
user_uf = st.selectbox("Selecione um estado para análise:",
utils.get_ufs_list())
prepare_heatmap(
br_cases,
place_type="city",
group=user_uf,
)
prepare_heatmap(
br_cases,
place_type="state",
)
prepare_heatmap(
loader.read_data(
"br",
config,
endpoint=config["br"]["api"]["endpoints"]["analysis"]["owid"]),
place_type="country_pt",
)
def get_score_groups(config, session_state, slider_value):
""" Takes our data and splits it into 4 sectors for use by our diagram generator """
# uf_num = utils.get_place_id_by_names(session_state.state)
if (session_state.city_name != "Todos"
or session_state.health_region_name != "Todos"):
endpoint = "health_region"
col = "health_region_id"
value = session_state.health_region_id
place_name = session_state.health_region_name + " (Região de Saúde)"
else:
endpoint = "state"
col = "state_num_id"
value = session_state.state_num_id
place_name = session_state.state_name + " (Estado)"
economic_data = loader.read_data(
"br",
config,
config["br"]["api"]["endpoints"]["safereopen"]["economic_data"]
[endpoint],
).query(f"{col} == {value}")
CNAE_sectors = loader.read_data(
"br", config,
config["br"]["api"]["endpoints"]["safereopen"]["cnae_sectors"])
CNAE_sectors = dict(zip(CNAE_sectors.cnae, CNAE_sectors.activity))
economic_data["activity_name"] = economic_data.apply(
lambda row: CNAE_sectors[row["cnae"]], axis=1)
return (
gen_sorted_sectors(
economic_data,
slider_value,
DO_IT_BY_RANGE,
),
economic_data,
place_name,
)
def plot_rt_wrapper(place_id, place_type):
endpoints = {
"state_num_id": "state",
"health_region_id": "health_region",
"city_id": "city",
}
data = (loader.read_data(
"br", config,
config["br"]["api"]["endpoints"]["rt"][endpoints[place_type]]).query(
f"{place_type} == {place_id}").sort_values("last_updated"))
if len(data) < 30:
return None
fig = plot_rt(data)
fig.update_layout(xaxis=dict(tickformat="%d/%m"))
fig.update_layout(margin=dict(l=50, r=50, b=100, t=20, pad=4))
fig.update_yaxes(automargin=True)
return fig
def train(args: Dict):
""" Train the NMT Model.
@param args (Dict): args from cmd line
:param args:
"""
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print('use device: %s' % device)
train_data, dev_data = read_data(args)
vocab, vocab_mask = read_vocab(args)
train_batch_size, N, d_model, d_ff, h, dropout, valid_niter, log_every, model_save_path, lr = read_model_params(
args)
transformer_model = TransfomerModel(vocab, N, d_model, d_ff, h, dropout,
device)
model = transformer_model.model
optimizer = NoamOpt(
model.src_embed[0].d_model, 1, 400,
torch.optim.Adam(model.parameters(),
lr=lr,
betas=(0.9, 0.999),
eps=1e-9))
# criterion = nn.CrossEntropyLoss()
# criterion = LabelSmoothing(size=len(vocab.tgt.word2id), padding_idx=vocab.tgt.word2id['<pad>'], smoothing=0.0)
criterion = LabelSmoothing(size=len(vocab.tgt.word2id),
padding_idx=vocab.tgt.word2id['<pad>'],
smoothing=0.001)
# criterion = partial(nllLoss, vocab.src.word2id['<pad>'])
loss_compute_train = SimpleLossCompute(model.generator, criterion,
optimizer)
loss_compute_dev = SimpleLossCompute(model.generator,
criterion,
optimizer,
train=False)
train_time = start_time = time.time()
patience = cum_loss = report_loss = cum_tgt_words = report_tgt_words = 0
num_trial = cum_exmaples = report_examples = epoch = valid_num = 0
hist_valid_scores = []
print('begin Maximum Likelihood Training')
while True:
epoch += 1
for train_iter, batch_sents in enumerate(
batch_iter(train_data,
batch_size=train_batch_size,
shuffle=True)):
loss, batch_size, n_tokens = train_step(model, batch_sents, vocab,
loss_compute_train, device)
report_loss += loss
cum_loss += loss
cum_exmaples += batch_size
report_examples += batch_size
report_tgt_words += n_tokens
cum_tgt_words += n_tokens
if train_iter % log_every == 0:
elapsed = time.time() - start_time
elapsed_since_last = time.time() - train_time
print(
f"epoch {epoch}, iter {train_iter}, avg loss {report_loss / report_examples: .3f}, "
f"avg ppl {np.exp(report_loss / report_tgt_words): .3f}, cum examples {cum_exmaples}, "
f"speed {report_tgt_words/ elapsed_since_last: .3f} w/s, elapsed time {elapsed: .3f} s, lr= {optimizer._rate}"
)
train_time = time.time()
report_tgt_words = report_loss = report_examples = 0.
if train_iter % valid_niter == 0:
print(
f"epoch {epoch}, iter {train_iter}, cum. loss {cum_loss/cum_exmaples}, "
f"cum ppl {np.exp(cum_loss / cum_tgt_words)}, cum exmples {cum_exmaples}, lr= {optimizer._rate}"
)
cum_loss = cum_exmaples = cum_tgt_words = 0.
valid_num += 1
print("begin validation ...")
dev_loss, dev_ppl = run_dev_session(model,
dev_data,
vocab,
loss_compute_dev,
batch_size=32,
device=device)
print(
f'validation: iter {train_iter}, dev. loss {dev_loss}, dev. ppl {dev_ppl}'
)
valid_metric = -dev_ppl
is_better = len(hist_valid_scores
) == 0 or valid_metric > max(hist_valid_scores)
hist_valid_scores.append(valid_metric)
if is_better:
patience = 0
print(
f'save currently the best model to {model_save_path}')
transformer_model.save(model_save_path)
torch.save(optimizer.optimizer.state_dict(),
model_save_path + ".optim")
elif patience < int(args['--patience']):
patience += 1
print(f'hit patience {patience}')
if patience == int(args['--patience']):
num_trial += 1
print(f"hit #{num_trial} trial")
if num_trial == int(args['--max-num-trial']):
print('early stop!')
exit(0)
if epoch == int(args['--max-epoch']):
print('reached max number of epochs!')
exit(0)
def main():
data = read_data()
data_frames = []
ctr = 0
length = len(data)
# Continue through the list until there is nothing left
while ctr < length:
data_frames.append(pd.DataFrame(np.transpose(data[ctr][2])))
ctr += 1
print("\n" + str(ctr) + " : Months Processed\n")
# Setup the final datafram with the columns
df = pd.concat(data_frames, ignore_index=True)
df.columns = VARNAMES
# Add in Label Column and make them 1 or 0
df["Label"] = df["Precipitation"] > 0
df.Label = df.Label.astype(int)
df = df.drop(columns=["Precipitation"])
print(df)
# Get rid of precipitation
correlation_matrix = df.corr()
print(correlation_matrix["Label"].sort_values(ascending=False))
# Create a stratified shuffle split for the data
split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
for train_index, test_index in split.split(df, df["Label"]):
strat_train = df.loc[train_index]
strat_test = df.loc[test_index]
# Prepare the Data for Machine Learning
weather_train = strat_train.drop("Label", axis=1)
weather_labels = strat_train["Label"].copy()
weather_test = strat_test.drop("Label", axis=1)
test_labels = strat_test["Label"].copy()
# Setup a Nearest-Neighbor
neigh2 = KNeighborsClassifier(n_neighbors=2)
neigh3 = KNeighborsClassifier(n_neighbors=3)
neigh5 = KNeighborsClassifier(n_neighbors=5)
neigh10 = KNeighborsClassifier(n_neighbors=10)
neigh2.fit(weather_train, weather_labels)
neigh3.fit(weather_train, weather_labels)
neigh5.fit(weather_train, weather_labels)
neigh10.fit(weather_train, weather_labels)
# Setup a Linear Model
lin_reg = LinearRegression()
lin_reg.fit(weather_train, weather_labels)
# Do Predictions
linear_predictions = lin_reg.predict(weather_test)
neigh2_pred = neigh2.predict(weather_test)
neigh3_pred = neigh3.predict(weather_test)
neigh5_pred = neigh5.predict(weather_test)
neigh10_pred = neigh10.predict(weather_test)
# Print out errors
lin_mae = mean_absolute_error(test_labels, linear_predictions)
neigh2_mae = mean_absolute_error(test_labels, neigh2_pred)
neigh3_mae = mean_absolute_error(test_labels, neigh3_pred)
neigh5_mae = mean_absolute_error(test_labels, neigh5_pred)
neigh10_mae = mean_absolute_error(test_labels, neigh10_pred)
# Print Statements for accuracy scores
print("Linear Accuracy Score: " + str(1 - lin_mae) + "%")
print("NN 2 accuracy Score : " + str(1 - neigh2_mae) + "%")
print("NN 3 accuracy Score : " + str(1 - neigh3_mae) + "%")
print("NN 5 accuracy Score : " + str(1 - neigh5_mae) + "%")
print("NN 10 accuracy Score : " + str(1 - neigh10_mae) + "%")
def main(user_input, indicators, data, config, session_state):
utils.genHeroSection(
title1="Onda",
title2="Covid",
subtitle=
"Veja e compare a evolução da curva de contágio da Covid-19 em seu estado ou município.",
logo="https://i.imgur.com/Oy7IiGB.png",
header=False)
try:
# load data
# print("loading br cases")
br_cases = loading_cached()
# print("finished laoding br cases")
my_dict = utils.Dictionary()
# ONDA POR ESTADO
da.prepare_heatmap(br_cases, place_type="state_id")
st.write("")
pass
except Exception as e:
st.write(str(e))
# ONDA POR MUNICIPIO
st.write(
"""
<div class="base-wrapper">
<span class="section-header primary-span">ONDA MORTES DIÁRIAS POR MUNICÍPIO</span>
<br><br>
<span class="ambassador-question"><b>Selecione seu estado e município para prosseguir</b></span>
</div>""",
unsafe_allow_html=True,
)
dfs, places_ids = get_data(loader.config)
state_name = st.selectbox("Estado ", utils.filter_place(dfs, "state"))
city_name = st.selectbox(
"Município ",
utils.filter_place(dfs,
"city",
state_name=state_name,
health_region_name="Todos"),
)
deaths_or_cases = (st.selectbox(
"Qual análise você quer ver: Número de mortes ou Taxa de letalidade (mortes por casos)?",
["Mortes", "Letalidade"]) == "Mortes por casos")
# print("checking")
if city_name != "Todos": # the user selected something
# print("passed")
br_cases = br_cases[br_cases["state_name"] ==
state_name] # .reset_index()
# gen_banners()
uf = my_dict.get_state_alphabetical_id_by_name(state_name)
da.prepare_heatmap(
br_cases,
place_type="city_name",
group=uf,
your_city=city_name,
deaths_per_cases=deaths_or_cases,
)
# print("finished preparation")
# ONDA POR PAÍS
st.write("")
da.prepare_heatmap(
loader.read_data(
"br",
loader.config,
endpoint=config["br"]["api"]["endpoints"]["analysis"]["owid"],
),
place_type="country_pt",
)
def main():
data = read_data()
data_frames = []
ctr = 0
length = len(data)
# Continue through the list until there is nothing left
while ctr < length:
data_frames.append(pd.DataFrame(np.transpose(data[ctr][2])))
ctr += 1
print("\n" + str(ctr) + " : Months Processed\n")
# Setup the final datafram with the columns
df = pd.concat(data_frames, ignore_index=True)
df.columns = VARNAMES
# Add in Label Column and make them 1 or 0
df["Label"] = df["Precipitation"] > 0
df.Label = df.Label.astype(int)
df = df.drop(columns=["Precipitation"])
print(df)
# Get rid of precipitation
correlation_matrix = df.corr()
print(correlation_matrix["Label"].sort_values(ascending=False))
# Create a stratified shuffle split for the data
split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
for train_index, test_index in split.split(df, df["Label"]):
strat_train = df.loc[train_index]
strat_test = df.loc[test_index]
# Prepare the Data for Machine Learning
weather_train = strat_train.drop("Label", axis=1)
weather_labels = strat_train["Label"].copy()
# Setup test set
weather_test = strat_test.drop("Label", axis=1)
test_labels = strat_test["Label"].copy()
# Setup a Linear Regression Model
lin_reg = LinearRegression()
lin_reg.fit(weather_train, weather_labels)
# Setup a Logistic Regression model
log_reg = LogisticRegressionCV(
cv=5,
random_state=0,
multi_class='multinomial',
)
log_reg.fit(weather_train, weather_labels)
# Do Predictions
linear_predictions = lin_reg.predict(weather_train)
linear_test_predictions = lin_reg.predict(weather_test)
logistic_predictions = log_reg.predict(weather_train)
logistic_test_predictions = log_reg.predict(weather_test)
lin_mae = mean_absolute_error(weather_labels, linear_predictions)
linear_test_mae = mean_squared_error(test_labels, linear_test_predictions)
log_mae = mean_absolute_error(weather_labels, logistic_predictions)
logistic_test_mae = mean_squared_error(test_labels,
logistic_test_predictions)
print("Linear MeanAbsoluteError on training data : " + str(1 - lin_mae) +
"%")
print("Linear MeanAbsoluteError on test data : " +
str(1 - linear_test_mae) + "%")
print("Logistic MeanAbsoluteError on training data: " + str(1 - log_mae) +
"%")
print("Logistic MeanAbsoluteError on test data : " +
str(1 - logistic_test_mae) + "%")
def main():
utils.localCSS("style.css")
utils.localCSS("icons.css")
# HEADER
utils.genHeroSection()
utils.genVideoTutorial()
# GET DATA
config = yaml.load(open('configs/config.yaml', 'r'),
Loader=yaml.FullLoader)
# if abs(datetime.now().minute - FIXED) > config['refresh_rate']:
# caching.clear_cache()
cities = loader.read_data('br', config, refresh_rate=refresh_rate(config))
# REGION/CITY USER INPUT
user_input = dict()
utils.genStateInputSectionHeader()
user_input['state'] = st.selectbox('Estado',
add_all(cities['state_name'].unique()))
cities_filtered = filter_options(cities, user_input['state'], 'state_name')
utils.genMunicipalityInputSection()
user_input['region'] = st.selectbox(
'Região SUS',
add_all(cities_filtered['health_system_region'].unique()))
cities_filtered = filter_options(cities_filtered, user_input['region'],
'health_system_region')
user_input['city'] = st.selectbox(
'Município', add_all(cities_filtered['city_name'].unique()))
cities_filtered = filter_options(cities_filtered, user_input['city'],
'city_name')
sources = cities_filtered[[
c for c in cities_filtered.columns
if (('author' in c) or ('last_updated_' in c))
]]
selected_region = cities_filtered.sum(numeric_only=True)
# GET LAST UPDATE DATE
if not np.all(cities_filtered['last_updated'].isna()):
last_update_cases = cities_filtered['last_updated'].max().strftime(
'%d/%m')
# GET NOTIFICATION RATE
if len(cities_filtered
) > 1: # pega taxa do estado quando +1 municipio selecionado
notification_rate = round(
cities_filtered['state_notification_rate'].mean(), 4)
else:
notification_rate = round(
cities_filtered['notification_rate'].values[0], 4)
# pick locality according to hierarchy
locality = choose_place(user_input['city'], user_input['region'],
user_input['state'])
st.write('<br/>', unsafe_allow_html=True)
utils.genInputCustomizationSectionHeader(locality)
# SOURCES USER INPUT
source_beds = sources[['author_number_beds',
'last_updated_number_beds']].drop_duplicates()
authors_beds = source_beds.author_number_beds.str.cat(sep=', ')
source_ventilators = sources[[
'author_number_ventilators', 'last_updated_number_ventilators'
]].drop_duplicates()
authors_ventilators = source_ventilators.author_number_ventilators.str.cat(
sep=', ')
if locality == 'Brasil':
authors_beds = 'SUS e Embaixadores'
authors_ventilators = 'SUS e Embaixadores'
user_input['n_beds'] = st.number_input(
f"Número de leitos destinados aos pacientes com Covid-19 (fonte: {authors_beds}, atualizado: {source_beds.last_updated_number_beds.max().strftime('%d/%m')})",
0, None, int(selected_region['number_beds']))
user_input['n_ventilators'] = st.number_input(
f"Número de ventiladores destinados aos pacientes com Covid-19 (fonte: {authors_ventilators}, atualizado: {source_ventilators.last_updated_number_ventilators.max().strftime('%d/%m')}):",
0, None, int(selected_region['number_ventilators']))
# POP USER INPUTS
user_input['population_params'] = {'N': int(selected_region['population'])}
user_input['population_params']['D'] = st.number_input(
'Mortes confirmadas:', 0, None, int(selected_region['deaths']))
# get infected cases
infectious_period = config['br']['seir_parameters'][
'severe_duration'] + config['br']['seir_parameters'][
'critical_duration']
if selected_region['confirmed_cases'] == 0:
st.write(f'''<div class="base-wrapper">
Seu município ou regional de saúde ainda não possui casos reportados oficialmente. Portanto, simulamos como se o primeiro caso ocorresse hoje.
<br><br>Caso queria, você pode mudar esse número abaixo:
</div>''',
unsafe_allow_html=True)
user_input['population_params']['I'] = st.number_input(
'Casos ativos estimados:', 0, None, 1)
else:
user_input['population_params']['I'] = int(
selected_region['infectious_period_cases'] / notification_rate)
st.write(f'''<div class="base-wrapper">
O número de casos confirmados oficialmente no seu município ou regional de saúde é de {int(selected_region['confirmed_cases'].sum())} em {last_update_cases}.
Dada a progressão clínica da doença (em média, {infectious_period} dias) e a taxa de notificação ajustada para a região ({int(100*notification_rate)}%),
<b>estimamos que o número de casos ativos é de {user_input['population_params']['I']}</b>.
<br><br>Caso queria, você pode mudar esse número para a simulação abaixo:
</div>''',
unsafe_allow_html=True)
user_input['population_params']['I'] = st.number_input(
'Casos ativos estimados:', 0, None,
user_input['population_params']['I'])
# calculate recovered cases
user_input = calculate_recovered(user_input, selected_region,
notification_rate)
# AMBASSADOR SECTION
utils.genAmbassadorSection()
st.write('<br/>', unsafe_allow_html=True)
# DEFAULT WORST SCENARIO
user_input['strategy'] = {'isolation': 90, 'lockdown': 90}
user_input['population_params']['I'] = [
user_input['population_params']['I']
if user_input['population_params']['I'] != 0 else 1
][0]
_, dday_beds, dday_ventilators = simulator.run_evolution(
user_input, config)
worst_case = SimulatorOutput(
color=BackgroundColor.GREY_GRADIENT,
min_range_beds=dday_beds['worst'],
max_range_beds=dday_beds['best'],
min_range_ventilators=dday_ventilators['worst'],
max_range_ventilators=dday_ventilators['best'])
# DEFAULT BEST SCENARIO
user_input['strategy'] = {'isolation': 0, 'lockdown': 90}
_, dday_beds, dday_ventilators = simulator.run_evolution(
user_input, config)
best_case = SimulatorOutput(
color=BackgroundColor.LIGHT_BLUE_GRADIENT,
min_range_beds=dday_beds['worst'],
max_range_beds=dday_beds['best'],
min_range_ventilators=dday_ventilators['worst'],
max_range_ventilators=dday_ventilators['best'])
resources = ResourceAvailability(locality=locality,
cases=selected_region['active_cases'],
deaths=selected_region['deaths'],
beds=user_input['n_beds'],
ventilators=user_input['n_ventilators'])
utils.genSimulationSection(int(user_input['population_params']['I']),
locality, resources, worst_case, best_case)
utils.genActNowSection(locality, worst_case)
utils.genStrategiesSection(Strategies)
st.write('''
<div class="base-wrapper">
<span class="section-header primary-span">Etapa 4: Simule o resultado de possíveis intervenções</span>
<br />
<span>Agora é a hora de planejar como você pode melhor se preparar para evitar a sobrecarga hospitalar. Veja como mudanças na estratégia adotada afetam a necessidade de internação em leitos.</span>
</div>''',
unsafe_allow_html=True)
user_input['strategy']['isolation'] = st.slider(
'Em quantos dias você quer acionar a Estratégia 2, medidas restritivas? (deixe como 0 se a medida já estiver em vigor)',
0,
90,
0,
key='strategy2')
user_input['strategy']['lockdown'] = st.slider(
'Em quantos dias você quer acionar a Estratégia 3, quarentena?',
0,
90,
90,
key='strategy3')
st.write('<br/><br/>', unsafe_allow_html=True)
# SIMULATOR SCENARIOS: BEDS & RESPIRATORS
fig, dday_beds, dday_ventilators = simulator.run_evolution(
user_input, config)
utils.genChartSimulationSection(
user_input['strategy']['isolation'],
user_input['strategy']['lockdown'],
SimulatorOutput(color=BackgroundColor.SIMULATOR_CARD_BG,
min_range_beds=dday_beds['worst'],
max_range_beds=dday_beds['best'],
min_range_ventilators=dday_ventilators['worst'],
max_range_ventilators=dday_ventilators['best']), fig)
utils.genWhatsappButton()
utils.genFooter()
bilstm = True
optim = 'adam'
device = 1
dropout = 0.5
penalty = 0.0
l2_reg = 0.0001
division = 25
seed = 283953214
fine_tune = None
#data_set = 'product/books,product/music'
if type(data_set) is not list:
data_set = data_set.split(',')
print(data_set)
train_data, dev_data, test_data, c, text_field, label_fields = loader.read_data(
data_set, batch_size)
#model_path = '../model/conll_dvd_electronics_kitchen_video_music_books_0.0_82.74.torch'
#model_path = '../model/Model.SingleAttn_sentiment_88.03.torch'
#model_path = '../model/old/Model.ASPCatAttn_mr_imdb_sports_soft_mag_baby_video_toys_music_health_camera_apparel_kitchen_dvd_elec_books_0.0001_0.0001_87.11.torch'
#model_path = '../model/Old_Model.AttnOverAttn_sentiment_87.73.torch'
model_path = '../model/Model.AttnOverAttn_topic_sentiment_46.85.torch'
model = torch.load(model_path)
#model = torch.load(model_path, map_location=lambda storage, loc: storage)
model.eval()
for i in range(len(data_set)):
acc = utils.evaluate(model,
test_data,
idx=i,
use_cuda=True,
def get_places_ids(config):
return loader.read_data("br", config, "br/places/ids")
def loading_cached_states():
return loader.read_data(
"br",
loader.config,
endpoint=loader.config["br"]["api"]["endpoints"]["analysis"]["cases_states"],
)
def main(ex_name,
data_set,
pre_emb,
vocab_size,
embed_dim,
lr,
use_model,
use_cuda,
n_epochs,
batch_size,
lstm_dim,
da,
r,
bilstm,
optim,
device,
_log,
_run,
l2_reg,
division,
fine_tune=None,
seed=283953214,
p_lambda=0.0,
dropout=0,
adversarial=False,
save_model=False,
transfer_params=False,
penalty=0.0,
params=[],
window_size=0,
epsilon=0.0,
search_n=0,
nlayers=1,
p_lambda2=0.0,
rl_batch=1,
p_gamma=1.0,
a_pen=0.0,
rec_acc=False,
lock_params=False):
_run.info['ex_name'] = ex_name
seed = 12345678
torch.manual_seed(seed)
data_set = data_set.split(',')
#train_data, dev_data, test_data, c, vocab_size, word_vector = loader.read_data(data_set, batch_size)
import dill
import pickle
import os
if len(data_set) == 0 and data_set[0] == 'imdb':
print('IMDB!')
file_path = '../data/aclImdb/data_%d.pickle' % batch_size
if os.path.exists(file_path):
print('reading IMDB pickle')
with open(file_path, 'r') as f:
train_data, dev_data, test_data, c, text_field, label_fields = pickle.load(
f)
else:
train_data, dev_data, test_data, c, text_field, label_fields = loader.read_data(
data_set, batch_size)
print('writing IMDB pickle')
with open(file_path, 'w') as f:
pickle.dump((train_data, dev_data, test_data, c, text_field,
label_fields), f)
else:
train_data, dev_data, test_data, c, text_field, label_fields = loader.read_data(
data_set, batch_size)
#exit()
lf = label_fields[-1]
print(lf.vocab.itos)
#exit(0)
vocab_size = len(text_field.vocab.itos)
word_vector = text_field.vocab.vectors
if adversarial and len(data_set) >= 15:
adv_train_data, _dev_data, _test_data, _c, _text_field, _label_fields = loader.read_data(
['product/topic'], batch_size)
model_param = OrderedDict()
model_param['vocab_size'] = vocab_size
model_param['embed_dim'] = embed_dim
model_param['lstm_dim'] = lstm_dim
model_param['da'] = da
if len(data_set) == 1:
model_param['c'] = c
else:
model_param['cs'] = c
model_param['batch_size'] = batch_size
model_param['r'] = r
model_param['bilstm'] = bilstm
model_param['device'] = device
model_param['dropout'] = dropout
model_param['params'] = params
model_param['window_size'] = window_size
model_param['epsilon'] = epsilon
model_param['p_lambda'] = p_lambda
model_param['p_lambda2'] = p_lambda2
model_param['p_gamma'] = p_gamma
model_param['search_n'] = search_n
model_param['nlayers'] = nlayers
model_param['rl_batch'] = rl_batch
model_param['a_pen'] = a_pen
model_param['data_set'] = data_set
if use_cuda and torch.cuda.is_available():
model_param['use_cuda'] = True
else:
use_cuda = False
model_param['use_cuda'] = False
_run.info['model_param'] = model_param
model = eval('Model.' + use_model)(model_param)
print(model)
_run.info['model'] = str(model)
params = list(model.parameters())
ans = 0
for p in params:
s = p.size()
if len(s) == 1:
ans += s[0]
else:
if s[0] >= 24739:
continue
ans += s[0] * s[1]
#print(params)
print(ans)
exit()
if model_param['use_cuda']:
model.cuda(device)
print("Initialize the word-embed layer")
model.init_embed(word_vector)
if transfer_params:
#model_from = torch.load('../model/copy/Model.MaxAttn_sentiment_86.77_copy.torch')
model_from = torch.load('../model/LM_imdb_151.torch')
#model_from = torch.load('../model/copy/Model.MaxAttn_sentiment_0.50_0.50_86.80.torch')
#model_from = torch.load('../model/copy/Model.MaxAttn_sentiment_0.00_1.00_87.17.torch')
#model_from = torch.load('../model/copy/Model.MaxAttn_sentiment_0.50_1.00_86.58.torch')
copy_param(model_from.encoder, model.embed)
model.embed.cuda(model.device)
copy_param(model_from.rnn, model.lstm)
model.lstm.cuda(model.device)
transfer_params = False
#lock_params = True
#lock_params = False
'''MaxAttn
copy_param(model_from.embed, model.embed)
model.embed.cuda(model.device)
copy_param(model_from.lstm, model.lstm)
model.lstm.cuda(model.device)
copy_param(model_from.attn, model.attn)
model.attn.cuda(model.device)
copy_param(model_from.mlp1, model.mlp1)
model.mlp1.cuda(model.device)
if 'unfix' in params:
transfer_params = False
'''
"""
'''RelateNet'''
model_from = torch.load('../model/Model.MultiSingleAttn_topic_sentiment__82.79.torch')
copy_param(model_from.embed, model.embed)
model.embed.cuda(model.device)
copy_param(model_from.lstm, model.lstm)
model.lstm.cuda(model.device)
copy_param(model_from.attn[0], model.attn1[0])
model.attn1[0].cuda(model.device)
copy_param(model_from.attn[1], model.attn1[1])
model.attn1[1].cuda(model.device)
if 'unfix' in params:
transfer_params = False
"""
'''
copy_param(model_from.embed, model.embed)
model.embed.cuda(model.device)
copy_param(model_from.lstm, model.lstm)
model.lstm.cuda(model.device)
copy_param(model_from.attn[0], model.attn[0])
model.attn[0].cuda(model.device)
copy_param(model_from.attn[1], model.attn[1])
model.attn[1].cuda(model.device)
#transfer_params = False
'''
#copy_param(model_from.attn[1], model.attn2)
#model.attn2.cuda(model.device)
stop = set([text_field.vocab.stoi[w] for w in ['.', '!', '?']])
print('fine_tine', fine_tune)
if fine_tune is not None:
model = torch.load('../model/' + fine_tune,
map_location=lambda storage, loc: storage)
model.cuda(device)
model.device = device
#utils.fine_tune(n_epochs, division, train_data, dev_data, model, lr, optim, use_cuda, l2_reg, data_set, p_lambda, _run)
#exit()
#utils.fine_tune(n_epochs, division, train_data, dev_data, test_data, model, lr, optim, use_cuda, l2_reg, data_set, p_lambda, p_gamma, save_model, adversarial, _run, lock_params=lock_params, penalty=penalty, params=params, stop=stop)
#exit()
if len(data_set) == 1:
utils.train(n_epochs, division, train_data, dev_data, test_data, model,
lr, optim, use_cuda, l2_reg, data_set, save_model, stop,
_run)
else:
if adversarial and len(data_set) >= 15:
utils.multi_train(n_epochs,
division,
train_data,
dev_data,
test_data,
model,
lr,
optim,
use_cuda,
l2_reg,
data_set,
p_lambda,
p_gamma,
save_model,
adversarial,
_run,
adv_train_iters=adv_train_data,
adv_label_field=_label_fields[0],
params=params)
else:
utils.multi_train(n_epochs,
division,
train_data,
dev_data,
test_data,
model,
lr,
optim,
use_cuda,
l2_reg,
data_set,
p_lambda,
p_gamma,
save_model,
adversarial,
_run,
lock_params=False,
penalty=penalty,
params=params,
stop=stop,
rec_acc=rec_acc)
def main():
data = read_data()
data_frames = []
ctr = 0
length = len(data)
# Continue through the list until there is nothing left
while ctr < length:
data_frames.append(pd.DataFrame(np.transpose(data[ctr][2])))
ctr += 1
print("\n" + str(ctr) + " : Months Processed\n")
# Setup the final datafram with the columns
df = pd.concat(data_frames, ignore_index=True)
df.columns = VARNAMES
# Add in Label Column and make them 1 or 0
df["Label"] = df["Precipitation"] > 0
df.Label = df.Label.astype(int)
df = df.drop(columns=["Precipitation"])
print(df)
# Get rid of precipitation
correlation_matrix = df.corr()
print(correlation_matrix["Label"].sort_values(ascending=False))
# Create a stratified shuffle split for the data
split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
for train_index, test_index in split.split(df, df["Label"]):
strat_train = df.loc[train_index]
strat_test = df.loc[test_index]
# Prepare the Data for Machine Learning
weather_train = strat_train.drop("Label", axis=1)
weather_labels = strat_train["Label"].copy()
weather_test = strat_test.drop("Label", axis=1)
test_labels = strat_test["Label"].copy()
# A couple of statements to get rid of verbosity (only shows errors, not warnings)
old_v = tf.logging.get_verbosity()
tf.logging.set_verbosity(
tf.logging.ERROR
) # old_v can be used to set_verbosity(old_v) back to old messages
# Setup the inputs
nn_inputs = 28 * 28
nn_hidden = 16
nn_outputs = 10
# Setup variables
X = tf.placeholder(tf.float32, shape=(None, nn_inputs), name="X")
y = tf.placeholder(tf.int64, shape=(None), name="y")
# Building the tf.name_scope
learning_rate = 0.01
# Implemented neural net using sigmoid function, found in activation parameter of hidden_1
with tf.name_scope("dnn"):
hidden_2 = tf.layers.dense(X,
nn_hidden,
name="hidden_2",
reuse=tf.AUTO_REUSE,
activation=tf.nn.sigmoid)
logits = tf.layers.dense(hidden_2,
nn_outputs,
reuse=tf.AUTO_REUSE,
name="outputs")
# Created loss function
with tf.name_scope("loss"):
xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=y, logits=logits)
loss = tf.reduce_mean(xentropy, name="loss")
# Implemented a gradient descent
with tf.name_scope("train"):
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
training_op = optimizer.minimize(loss)
# Evaluation Functions
with tf.name_scope("eval"):
correct = tf.nn.in_top_k(logits, y, 1)
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
init = tf.global_variables_initializer()
saver = tf.train.Saver()
# Set Epoch and Batch Size
epochs = 40
batch_size = 50
# Begin evaluation
with tf.Session() as sess:
init.run()
for epoch in range(epochs):
sess.run(training_op,
feed_dict={
X: weather_train,
y: weather_labels
})
# Report the training accuracies
training_acc = accuracy.eval(feed_dict={
X: weather_train,
y: weather_labels
})
validation_acc = accuracy.eval(feed_dict={
X: weather_test,
y: test_labels
})
print("Epoch :" + str(epoch) + "\n[Train Accuracy] : " +
str(training_acc) + "\n[Validation Accuracy] :" +
str(validation_acc))
def main():
data = read_data()
data_frames = []
ctr = 0
length = len(data)
# Continue through the list until there is nothing left
while ctr < length:
data_frames.append(pd.DataFrame(np.transpose(data[ctr][2])))
ctr += 1
print("\n" + str(ctr) + " : Months Processed\n")
# Setup the final datafram with the columns
df = pd.concat(data_frames, ignore_index=True)
df.columns = VARNAMES
# Add in Label Column and make them 1 or 0
df["Label"] = df["Precipitation"] > 0
df.Label = df.Label.astype(int)
df = df.drop(columns=["Precipitation"])
print(df)
# Get rid of precipitation
correlation_matrix = df.corr()
print(correlation_matrix["Label"].sort_values(ascending=False))
# Create a stratified shuffle split for the data 80/20 for train/test
split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
for train_index, test_index in split.split(df, df["Label"]):
strat_train = df.loc[train_index]
strat_test = df.loc[test_index]
# Prepare the Data for Machine Learning
weather_train = strat_train.drop("Label", axis=1)
weather_labels = strat_train["Label"].copy()
# Setup test set
weather_test = strat_test.drop("Label", axis=1)
test_labels = strat_test["Label"].copy()
# Setup a Linear Regression model for comparison
lin_reg = LinearRegression()
lin_reg.fit(weather_train, weather_labels)
# Setup a Grid Searched SVM
params = best_parameters(weather_train, weather_labels, 5)
support_vector = svm.SVC(kernel=params['kernel'],
C=params['C'],
gamma=params['gamma'])
support_vector.fit(weather_train, weather_labels)
# Do Linear Predictions
linear_predictions = lin_reg.predict(weather_train)
linear_test_predictions = lin_reg.predict(weather_test)
# Do SVM Predictions
svm_predictions = support_vector.predict(weather_train)
svm_test_predictions = support_vector.predict(weather_test)
# Print out errors
lin_mae = mean_absolute_error(weather_labels, linear_predictions)
lin_test_mae = mean_absolute_error(test_labels, linear_test_predictions)
svm_mae = mean_absolute_error(weather_labels, svm_predictions)
svm_test_mae = mean_absolute_error(test_labels, svm_test_predictions)
print("Linear MeanAbsoluteError on train: " + str(1 - lin_mae) + "%")
print("Linear MeanAbsoluteError on test : " + str(1 - lin_test_mae) + "%")
print("SVM MeanAbsoluteError on train : " + str(1 - svm_mae) + "%")
print("SVM MeanAbsoluteError on test : " + str(1 - svm_test_mae) + "%")
