def load_data_callback(country_in):
app.server.logger.info('load_data_callback')
if not country_in:
raise PreventUpdate
load_data(country_in, all_data_dict, app)
region_drp, province_drp = data2dropdown(country_in, all_data_dict)
return region_drp, province_drp, 'done'
def run_model(self, datafile):
"""Run the prep-fews model
Returns a dict witht the different performance measures
"""
time_series = fun.load_data(datafile)
for event in time_series['Runoff']:
# calculate losses if no forecasting system is operational
climate_loss = self.estimate_damage(event)
# calculate losses if a forecasting system is operational
forecast = self.issue_forecast(event)
warning_outcome = self.assess_warning_outcome(event, forecast)
warning_damage = self.get_warning_damage(event, warning_outcome,
self.preparedness)
warning_loss = self.get_warning_loss(event, warning_outcome,
self.preparedness)
# keep track of the states of the model
self.forecast_ls.append(forecast)
self.outcome_ls.append(warning_outcome)
self.warning_loss_ls.append(warning_loss)
self.climate_loss_ls.append(climate_loss)
self.warning_damage_ls.append(warning_damage)
self.preparedness_ls.append(self.preparedness)
# update the variables
if self.ct_prep is False:
self.preparedness = self.update_preparedness(
warning_damage, self.preparedness)
def data() -> list:
name_file = input("Wprowadź nazwę pliku: ")
m = functions.load_data(name_file)
print("Twoja macierz:")
for s in m:
print("wybór", *s)
return m
def main():
start_time = time.time()
in_arg = args_input()
# get the data
dataset, dataloaders = functions.load_data()
# get the model
model = functions.get_model(in_arg.arch)
# get the classifier, criterion, optimizer, device
model, classifier, criterion, optimizer, device = network_param(
model, in_arg.arch, in_arg.hidden_units, in_arg.learning_rate,
in_arg.gpu)
model.to(device)
print('Training model...')
functions.train_model(model, criterion, optimizer, device,
dataloaders['train'], dataloaders['val'],
in_arg.epochs)
# validation on test data
print('\nGetting results on test data accuracy...')
functions.test_model(model, criterion, device, dataloaders['test'])
# saving checkpoint on trained model
print('\nSaving checkpoint for current trained model...')
functions.save_checkpoint(model, optimizer, dataset, in_arg.arch,
in_arg.epochs, in_arg.save_dir)
print('Checkpoint saved!')
def __init__(self, name):
print(name)
self.name = name
self.scores, self.best_scores, self.mean_scores, self.std_scores = functions.load_data(self.name)
self.N_population = 2
self.last_gen = []
for i in range(self.N_population):
self.last_gen.append(torch.load(self.name+"/TorchSaves/Last_Gen"+str(998+i)+".pt"))
def data():
name_file = input("Wprowadź nazwę pliku: ")
m = functions.load_data(name_file)
print("Twoja macierz:")
print("A/B")
for s in m:
print(*s)
return m
def main():
data, optimal = f.load_data()
params = f.load_parameters()
print("Optimal:: {}\n".format(optimal))
der_uebermensch, fitness_dictionary, error = genetik.NaturalSelection(
len(data), params, data, optimal)
print(der_uebermensch)
if params["graph_data"] is True:
f.show_graph(fitness_dictionary, params, error)
else:
quit()
def __removing_from_blacklist(self):
"""Удаление пользователя с чёрного списка"""
def remove_user(uid, black_list):
"""Удаление пользователя"""
if uid in black_list:
black_list.remove(uid)
# уведомление
message = "\n - Пользователь успешно удалён!"
print(message + "\n")
# логирование
functions.logging('{} Удалён (ID): {}'.format(message, uid),
'Отсутствует', 'Успешное удаление пользователя с чёрного листа')
else:
print("\n - Пользователь с данным ID не найден!\n")
time.sleep(3)
def get_id_from_console():
"""Получение ID пользователя с консоли, который будет удалён """
try:
got_id = int(input("\nВведите ID пользователя, которого хотите удалить: "))
if len(str(got_id)) == 9:
return got_id
else:
print("\n - ID должен состоять из 9 чисел")
return None
except ValueError:
print("\n - ID должен состоять только из чисел")
while True:
try:
blacklist = functions.load_data(self.BLACKLIST)
except FileNotFoundError as error:
message = "\n - Файл не найден!"
print(message + "\n")
functions.logging(message, error)
time.sleep(3)
else:
user_id = get_id_from_console()
if user_id:
# удаление пользователя
remove_user(user_id, blacklist)
# обновление чёрного списка
self.__registration('w', self.BLACKLIST, blacklist)
break
def __init__(self, df = None, file_id = "", strategy = "momentum_trading"):
if df is None:
self.df = f.load_data()
else:
self.df = df
self.file = file_id
self.df_rt = f.make_returns(self.df)
self.df_lrt = f.make_log_returns(self.df)
self.df_lrt_30ma = f.get_moving_average(self.df_lrt)
self.indices = self.df_lrt.index
self.n = len(self.df_lrt)
self.tradecosts = 0.001
self.strategy = strategy
self.assets = self.df.columns
self.endowment = np.random.choice(self.assets, 5, replace = False)
self.startperiod = 31
self.cur_date_ind = self.startperiod
self.start_date = self.indices[self.cur_date_ind]
self.current_date = self.start_date
self.cur_return = 0
self.baseline_return = 0
self.tradefreq = 14
self.cash = 0
self.cash_baseline = 0
self.finish = False
self.endowment = f.get_worst_performers(self.df_lrt_30ma, self.current_date)
self.cash -= np.sum(self.df.loc[self.current_date, self.endowment] * (1 + self.tradecosts))
self.cash_baseline -= np.sum(self.df.loc[self.current_date, :]) * (1 + self.tradecosts)
## current value when cashing out
self.cur_value = self.cash + np.sum(self.df.loc[self.current_date, self.endowment]) * (1 - self.tradecosts)
self.cur_value_baseline = self.cash_baseline + np.sum(self.df.loc[self.current_date, :]) * (1 - self.tradecosts)
self.overall_return = 1
self.overall_return_base = 1
self.results = []
self.threshold = None
def main_definition():
"""Определение управляющего бота, регистрации, автоответчика"""
try:
configs = functions.load_data('data/config-bot.json')
bot_token, my_id = configs['TOKEN'], configs['my_ID']
except FileNotFoundError as error:
message = "\n - Файл с конфигурациями для бота не найден! Зарегестрируйте данные!"
print(message + "\n")
functions.logging(message, error)
functions.do_command()
else:
bot = TeleBot(bot_token)
bot.remove_webhook()
registration = Register()
answering_bot = AnsweringBot()
if not os.path.exists(registration.LOGS):
functions.creating_file(registration.LOGS)
return bot, registration, answering_bot, my_id
import psycopg2
import seaborn as sns
from sklearn.feature_extraction.text import CountVectorizer
sklearn.feature_extraction import text
## TODO: get rid of:
import common
import functions as f
matplotlib.style.use('ggplot')
## TODO: fix unicode here.
df = f.load_data()
# ## Examining and cleaning data
# Extract features
# TODO: remove stopwords before vectorizing, because the stop words don't capture bigrams.
## this is for
# In[14]:
## this model, with 40 topics, is not bad.
m = lda.LDA(n_topics=40, random_state=0, n_iter=200)
#print 'no data willbe plotted...'
plotdata=dd
elif (dd=='yes') or (args.VAL==[]):
#print 'plot everything...'
plotdata='yes'
else:
sys.exit("Error (-plotdata): Only 'yes' or 'no' values are admisible.")
#######################
### MAIN ##############
#######################
sys.stdout.write("\a")
#loading data points ('N' points for each one of the 'DIM' CVs)
N=file_lines(args.INPUT)
DIM=file_cols(args.INPUT)
P1=load_data(args.INPUT,N,DIM)
dK1=[]
dK2=[]
conv_all=[]
# estimate the initial string (P2) of size Min
P2 = string_ini(P1,DIM,args.K)
Pini=string_ini(P1,DIM,args.K)
PiniT=zip(*Pini)
# maximun distances of the data to the images of the string
maxdistances=maxvec(P1,P2)
# Estimate the mean of the voronoid cells
P3, fail, free_energy, allpoints, closeP, closeI = voroid_mean(P1,P2,DIM,maxdistances,0)
def __init__(self, nb_epochs, initial_lr):
self.epochs = nb_epochs
self.initial_lr = initial_lr
def __call__(self, epoch_idx):
if epoch_idx < self.epochs * 0.25:
return self.initial_lr
elif epoch_idx < self.epochs * 0.50:
return self.initial_lr * 0.2
elif epoch_idx < self.epochs * 0.75:
return self.initial_lr * 0.04
return self.initial_lr * 0.008
# Load Base
image, gender, age = ft.load_data("serializer/MATimdb64.mat")
x_data = image
y_data_g = np_utils.to_categorical(gender, 2)
y_data_a = np_utils.to_categorical(age, 101)
# Fatores importantes:
# optimizers, largura da rede, profundidade da rede, losses, val_split,
# model WidesResNet (WRN) dupla saida(age,gender)
model = WideResNet(64, depth=16, k=8)()
#model.summary()
#optimizers
model.compile(SGD(lr=0.1, momentum=0.9, nesterov=True),
loss=['categorical_crossentropy', 'categorical_crossentropy'],
from functions import get_best_wide, load_results, aggregate_results, load_data, resample_ohlc, hma_calc_old
import math
pair = 'TOMOBTC'
strat = 'hma_strat'
timescale = '1h'
train = 2000
test = 50
train_string = f'{train}-{test}'
params = 'lengths5-501-2'
metric = 'sqn'
print('- Loading data')
price, vol = load_data(pair)
print('- Resampling data')
price, vol = resample_ohlc(price, vol, timescale)
print(f'len(price): {len(price)}')
print('- Loading results')
df_dict = load_results(strat, pair, timescale, train_string, params)
print('- Running get_best')
best = get_best_wide(metric, df_dict, train, test)
print('Running hma_calc')
hma_25 = hma_calc_old(price, 25)
print('Running hma_calc')
hma_327 = hma_calc_old(price, 327)
nan_count = 0
for q in range(len(hma_25)):
def SM_test(summaryMeasure,
k,
data,
label,
data_test,
label_test,
batch_size,
is_mmens=False):
train_split_k = train_split[k]
val_split_k = val_split[k]
#reset the model before new cv
model = CNN_model()
model.to(device)
# Loss and optimizer
criterion = nn.BCEWithLogitsLoss()
optimizer = optim.SGD(model.parameters(),
momentum=0.9,
lr=0.001,
weight_decay=1e-3)
# load data
train_split_k = train_split[k]
testData = load_data(
split='test',
train_split=train_split_k,
val_split=val_split_k,
summaryMeasure=summaryMeasure,
data=data,
label=label, # need train data for mean and sd normalization
data_test=data_test,
label_test=label_test)
test_loader = DataLoader(dataset=testData,
batch_size=batch_size,
shuffle=True)
# set and load best model
model = CNN_model()
model.load_state_dict(
torch.load(MODEL_STORE_PATH + '/' + summaryMeasure + '_BEST_model' +
'_CV' + str(k + 1) + '.pt'))
model.to(device)
# START TRAINING FOR THIS CV
# TEST EVALUATION
model.eval()
total_ep = 0.0
correct_ep = 0.0
loss_ep = 0.0
current_ep_acc = 0.0
current_ep_loss = 0.0
total_step_v = len(test_loader)
# F1 calc
F1_labels = []
F1_pred = []
for i, (images, labels
) in enumerate(test_loader): #TEST THE MODEL ON THE VALIDATION
if device.type == "cuda":
images = images.to(device)
labels = labels.to(device)
labels = labels.reshape(len(labels), 1).type(torch.cuda.FloatTensor)
outputs_v = model(images) # the forward uses the entire batch together
predicted = outputs_v.data > 0.0 #define True if >0.5 and False if <0.5
# collect true labels and predicted for metrix calc
if i == 0:
F1_labels = labels.int().cpu().numpy()
F1_pred = predicted.int().cpu().numpy()
else:
F1_labels = np.concatenate((F1_labels, labels.int().cpu().numpy()))
F1_pred = np.concatenate((F1_pred, predicted.int().cpu().numpy()))
loss_v = criterion(outputs_v, labels)
total_v = labels.size(
0) # get it in case the last batch has different number of sample
correct_v = (predicted == labels
).sum().item() # sum the correct answers for this batch
#acc_list_v.append(correct_v / total_v) # n of correct / n of sample in this batch
total_ep += total_v # sum all the samples. it must end up being the tot training set
correct_ep += correct_v # sum together the right answers in entire training set
loss_ep += (loss_v.item() * len(labels))
current_ep_acc = (correct_ep / total_ep) * 100
current_ep_loss = loss_ep / total_ep
acc = accuracy_score(F1_labels, F1_pred)
bal_acc = balanced_accuracy_score(F1_labels, F1_pred)
F1_Score = f1_score(F1_labels, F1_pred,
average='weighted') #, zero_division=1)
tn, fp, fn, tp = confusion_matrix(F1_labels, F1_pred).ravel()
print('CV {}, sumMeasure: {} Test Accuracy of the model is: {}, F1: {}%'.
format(k + 1, summaryMeasure, current_ep_acc, F1_Score))
if is_mmens == True:
return F1_labels.reshape(-1), F1_pred.reshape(-1)
else:
return bal_acc, f1_score
def SM_train_val_kfoldCV(summaryMeasure, data, label, batch_size, num_epochs,
n_split):
# CROSSVALIDATION LOOP
for k in np.arange(n_split):
train_split_k = train_split[k]
val_split_k = val_split[k]
#reset the model before new cv
model = CNN_model()
model.to(device)
# Loss and optimizer
criterion = nn.BCEWithLogitsLoss()
optimizer = optim.SGD(model.parameters(),
momentum=0.9,
lr=0.001,
weight_decay=1e-3)
# load data
trainData = load_data(split='train',
train_split=train_split_k,
val_split=val_split_k,
summaryMeasure=summaryMeasure,
data=data,
label=label)
valData = load_data(split='val',
train_split=train_split_k,
val_split=val_split_k,
summaryMeasure=summaryMeasure,
data=data,
label=label)
train_loader = DataLoader(dataset=trainData,
batch_size=batch_size,
shuffle=True)
val_loader = DataLoader(dataset=valData,
batch_size=batch_size,
shuffle=True)
# START TRAINING FOR THIS CV
# collect loss and acc of each epoch
epoch_loss_train = []
epoch_acc_train = []
epoch_loss_val = []
epoch_balacc_val = []
epoch_F1_Score = []
best_acc = None
# TRAIN THE MODEL
for epoch in range(num_epochs):
total_ep = 0.0
correct_ep = 0.0
loss_ep = 0.0
model.train()
for i, (images, labels) in enumerate(train_loader):
if device.type == "cuda":
images = images.to(device)
labels = labels.to(device)
labels = labels.reshape(len(labels),
1).type(torch.cuda.FloatTensor)
# Run the forward pass
outputs = model(images)
loss = criterion(outputs, labels)
# Backprop and perform optimisation
optimizer.zero_grad()
loss.backward()
optimizer.step()
# Track the accuracy
total = labels.size(0)
predicted = outputs.data > 0.0
correct = (predicted == labels).sum().item()
total_ep += total
correct_ep += correct
loss_ep += (loss.item() * len(labels))
epoch_acc_train.append((correct_ep / total_ep))
epoch_loss_train.append(loss_ep / total_ep)
# Test the model on validation data
model.eval()
total_ep = 0.0
correct_ep = 0.0
loss_ep = 0.0
# F1 calc
F1_labels = []
F1_pred = []
for i, (images, labels) in enumerate(val_loader):
if device.type == "cuda":
images = images.to(device)
labels = labels.to(device)
labels = labels.reshape(len(labels),
1).type(torch.cuda.FloatTensor)
outputs_v = model(
images) # the forward uses the entire batch together
predicted = outputs_v.data > 0.0 #define True if >0.5 and False if <0.5
# collect true labels and predicted for metrix calc
if i == 0:
F1_labels = labels.int().cpu().numpy()
F1_pred = predicted.int().cpu().numpy()
else:
F1_labels = np.concatenate(
(F1_labels, labels.int().cpu().numpy()))
F1_pred = np.concatenate(
(F1_pred, predicted.int().cpu().numpy()))
loss_v = criterion(outputs_v, labels)
total_v = labels.size(0)
correct_v = (predicted == labels).sum().item()
total_ep += total_v
correct_ep += correct_v
loss_ep += (loss_v.item() * len(labels))
# calculate metrices of this epoch
current_ep_loss = loss_ep / total_ep
current_ep_acc = (correct_ep / total_ep)
acc = accuracy_score(F1_labels, F1_pred)
balacc = balanced_accuracy_score(F1_labels, F1_pred)
F1_Score = f1_score(F1_labels, F1_pred, average='weighted')
tn, fp, fn, tp = confusion_matrix(F1_labels, F1_pred).ravel()
if not best_acc or best_acc < current_ep_acc:
best_acc = current_ep_acc
torch.save(
model.state_dict(),
MODEL_STORE_PATH + '/' + summaryMeasure + '_BEST_model' +
'_CV' + str(k + 1) + '.pt')
# collect matrixes of all the epochs
epoch_loss_val.append((loss_ep / total_ep))
epoch_balacc_val.append(balacc)
print('{}, CV_{}, best accuracy = {}'.format(summaryMeasure, k + 1,
acc))
# path to data that has been cropped into smaller same-sized chucks
cropped_ground_truth_path_train = '/run/media/henryp/HenryHDD/DataSets/CMU/Formatted/Train/Cropped_data/sliced_'
cropped_ground_truth_path_test = '/run/media/henryp/HenryHDD/DataSets/CMU/Formatted/Test/Cropped_data/'
if run_on == 'GPU':
device = torch.device('cuda:0')
else:
device = torch.device('cpu')
print('============')
print('Training on: ' + str(device))
print('============')
training_partition, training_set_size = load_data(
ground_truth_dir=cropped_ground_truth_path_train, type='train')
training_set = DataSet(list_IDS=training_partition['train'],
data_dir=cropped_ground_truth_path_train,
clip_length=64)
training_generator = data.DataLoader(training_set, **params)
testing_partition, testing_set_size = load_data(
ground_truth_dir=cropped_ground_truth_path_test, type='test')
testing_set = DataSet(list_IDS=testing_partition['test'],
data_dir=cropped_ground_truth_path_test,
clip_length=64)
testing_generator = data.DataLoader(testing_set, **params)
help='Turn GPU mode on or off, default set to "off".')
results = parser.parse_args()
data_dir = results.data_directory
save_dir = results.save_directory
learning_rate = results.l_rate
dropout_rate = results.dropout_rate
hidden_unit_count = results.hidden_units
epochs = results.num_epochs
gpu_mode = results.gpu
arch = results.pretrained_model
# Load and preprocess data
train_loader, test_loader, validate_loader, train_data, test_data, validate_data = load_data(
data_dir)
# Load pretrained model
pre_train_model = results.pretrained_model
model = getattr(models, pre_train_model)(pretrained=True)
# Build and attach new classifier
input_unit_count = model.classifier[0].in_features
build_classifier(model, input_unit_count, hidden_unit_count, dropout_rate)
# Using a NLLLoss as output is LogSoftmax
criterion = nn.NLLLoss()
# Using Adam optimiser algorithm - uses concept of momentum to add fractions
# to previous gradient descents to minimize loss function
optimizer = optim.Adam(model.classifier.parameters(), learning_rate)
# constants
W = 20
f = 1
def usage():
print "Usage:\n%s [cross_val|simulation]" % sys.argv[0]
if len(sys.argv) < 2:
usage()
if sys.argv[1] == 'cross_val':
CV_LEN = 10
total_df = load_data(ALL_DAYS[:W + CV_LEN],
ignore_auctions=True,
correct_zero_volumes=True,
num_minutes_interval=1)
lambdas = [np.inf]
for b in range(1, 10):
rolling_simulator(total_df,
lambdas,
W,
analyze_day,
volume_model_meta_params={
'num_factors': f,
'bandwidth': b
})
total_df['DynamicSolinf_b_%d' % b] = total_df.DynamicSolinf
return parser
#--------------------------------
_p = _make_parser()
__doc__ += _p.format_help()
######################################################
if __name__ == '__main__':
args = _p.parse_args()
#######################################
############ CHECK_PCURVE ################
#######################################
#File with the pcurv estate (output from pcurve.py)
## loading pcurve state
N, DIM, Min, Mout, smooth, Niter, Ps, Psout, converg, free_energy, allpoints, closeP, closeI, closePP, closeII = loadpcurve(args.file1)
P1=load_data(args.file2,N,DIM)
print '#######################################################'
print '############# pcurve run parameters ##################'
print '#######################################################'
print ' '
print 'N (Size of the CV vectors) = ',N
print 'DIM (# of CVs) = ',DIM
print 'Min (#of points on the string) = ',Min
print 'Mout (# of final reparametrized points) = ',Mout
print 'smooth (strength of the smoothing [0-1]) = ',smooth
print 'Niter (# of iterations) = ',Niter
print ' '
print '#######################################################'
if args.choice==0:
PPP=Ps
def __removing_from_user_list(self):
"""Удаление пользователя со списка друзей"""
def find_key_via_value(uid, got_list):
"""Поиск ключа с помощью значения, для получения пользователя"""
if type(uid) == int:
if uid in got_list.values():
i = 0
for value in got_list.values():
if value == uid:
return list(got_list.keys())[i]
else:
i += 1
else:
return None
else:
return uid
def get_user_from_console():
"""Получение данных (ID или Имя) пользователя, который будет удалён со списка друзей"""
got_user = input("\nВедите ID или имя пользователя, которого хотите удалить: ")
try:
got_user = int(got_user)
if len(str(got_user)) == 9:
return got_user
else:
print("\n - ID должен состоять из 9 чисел")
return None
except ValueError:
return got_user
def remove_user(user_id, users_list):
person = find_key_via_value(user_id, users_list)
if not person:
print("\n - Пользователь с данным ID не найден!")
else:
if person.capitalize() in users_list.keys():
del users_list[person.capitalize()]
# уведомление
message = "\n - Пользователь успешно удалён!"
print(message + "\n")
# логирование
functions.logging('{} Удалён: {}'.format(message, person.capitalize()),
'Отсутствует', 'Успешное удаление пользователя со списка друзей')
else:
print("\n - Пользователь с данным Именем не найден!")
time.sleep(3)
while True:
try:
user_list = functions.load_data(self.USERS)
except FileNotFoundError as error:
message = "\n - Файл не найден"
print(message + "\n")
# логирование
functions.logging(message, error)
time.sleep(3)
else:
user = get_user_from_console()
if user:
# удаление пользователя
remove_user(user, user_list)
# обновление списка друзей
self.__registration('w', self.USERS, user_list)
break
import functions as f
import pandas as pd
import numpy as np
from sklearn.preprocessing import MinMaxScaler
import matplotlib.pyplot as plt
import seaborn as sns
from pylab import rcParams
from sklearn.model_selection import train_test_split
from keras.models import load_model
import tensorflow as tf
rcParams['figure.figsize'] = 12, 6
sns.set(style='whitegrid', palette='muted', font_scale=1.5)
path = "dados\justica_eleitoral.json"
dataLoad= f.load_data(path)
encoded_seqs = f.prepare_inputs(dataLoad)
autoencoder = load_model('dados\model_seqs2.h5')
# ### Carregando novamente a base de dados com as anomalias
scaler = MinMaxScaler()
scaled_seqs = scaler.fit_transform(encoded_seqs)
# fazer a predição com base na rede treinada
predicted = autoencoder.predict(scaled_seqs)
# MSE em termo de erro
mse = np.mean(np.power(scaled_seqs - predicted, 2), axis=1)
#Data Frame com MSE
st.markdown(intro_markdown, unsafe_allow_html=True)
st.markdown("---")
# Sidebar inputs
st.sidebar.header('k-means clustering')
st.sidebar.write(
'Select a date range and the number of clusters, then click Run to see a preview of the clustered data'
)
start_date = st.sidebar.date_input('Start date', date(2018, 1, 1))
end_date = st.sidebar.date_input('End date', date.today())
slider_input = st.sidebar.slider('Number of clusters (k)',
min_value=1,
max_value=10)
# Load cleaned data
runs = load_data()
# k-means clustering
if st.sidebar.button('Run'):
output = clustering(runs, start_date, end_date, slider_input)
cluster_counts = output[0]
st.sidebar.write('Clusters')
st.sidebar.dataframe(cluster_counts)
st.write('Preview')
bucket_data = output[1]
st.dataframe(bucket_data.head())
st.sidebar.markdown(filedownload(bucket_data), unsafe_allow_html=True)
st.sidebar.markdown("---")
if gpu_mode:
print('GPU mode is selected')
if torch.cuda.is_available() is False:
print('please check the hardware. No GPU is detected')
while input("Do you want to continue? [y/n]") == "n":
exit()
else:
print('Program will train on CPU .. will take longer time')
while input("Do you want to continue running on CPU mode? [y/n]") == "n":
exit()
print('... dont switch off computer .. training..')
start = time()
# working on data .. loading processing.. converting it into tensor etc..
trainloader, testloader, validloader, train_data, test_data, valid_data = load_data(
data_dir)
# loading a pretrained model
pre_tr_model = results.pretrained_model
model = getattr(models, pre_tr_model)(pretrained=True)
# building a new classifier and replacing it with old classifier
input_units = model.classifier[0].in_features
build_classifier(model, input_units, hidden_units, dropout)
# criterion.. NLLoss selected since we provided logsoftmax for classifier
criterion = nn.NLLLoss()
# optimizer
optimizer = optim.Adam(model.classifier.parameters(), learning_rate)
# train model
from functions import visualization_cluster
from functions import load_data
from sklearn.cluster import DBSCAN
from sklearn.manifold import Isomap
# load data
_, times, _, _, duration = load_data()
# convert feature times and duration into ms
duration = [d / 1e6 for d in duration]
times = [[t / 1e6 for t in sample] for sample in times]
# remove 0.14 and 4.22 are most likely to be measurment errors
for _ in range(2):
index = duration.index(min(duration))
del times[index]
del duration[index]
model = DBSCAN(eps=25, min_samples=100, n_jobs=-1, algorithm='ball_tree')
# visualization with Isomap
viz_model = Isomap(n_neighbors=15, n_components=2, n_jobs=-1)
cluster = model.fit_predict(times)
visualization_cluster(times, cluster, duration, viz_model, 25000,
"features_duration_isomap")
#%%
import pandas as pd
import numpy as np
import fbprophet as prop
import functions
import matplotlib.pyplot as plt
import os
import csv
import glob
import datetime
import joblib
import time
#%%
#get our data
df_train, df_test = functions.load_data()
#%%
#create the group by object
groups = df_train.groupby(['location', 'payment_type'])
forecasts = pd.DataFrame()
print('processing ' + str(len(groups)) + ' partitions ')
#%%
s = time.time()
for group_name, g in groups:
location = str(g['location'].unique())
payment_type = str(g['payment_type'].unique())
dest="hidden_units",
default=256,
type=int)
parser.add_argument('--dropout', dest="dropout", default=0.2)
parser.add_argument('--learning_rate', dest="learning_rate", default=0.001)
parser.add_argument('--gpu', dest="gpu", default=True, type=bool)
parser.add_argument('--epochs', dest="epochs", default=2, type=int)
parser.add_argument('--print_every', dest="print_every", default=10, type=int)
parser.add_argument('--save_dir',
dest="save_dir",
default="./checkpoint.pth",
type=str)
parsed = parser.parse_args()
data_dir = parsed.data_dir
arch = parsed.arch
hidden_units = parsed.hidden_units
dropout = parsed.dropout
learning_rate = parsed.learning_rate
gpu = parsed.gpu
epochs = parsed.epochs
print_every = parsed.print_every
save_dir = parsed.save_dir
trainloader, validloader, class_to_idx = functions.load_data(data_dir)
model, criterion, optimizer = functions.setup(arch, hidden_units, dropout,
learning_rate, gpu)
functions.train_network(model, criterion, optimizer, trainloader, validloader,
epochs, print_every, gpu)
functions.save_checkpoint(save_dir, arch, hidden_units, dropout, learning_rate,
gpu, model, optimizer, class_to_idx)
result['Sigma'], result['S_orig'], result['U'] = \
self._computeSigma()
else:
result['Sigma'] = self._computeSigma()
return result
if __name__ == "__main__":
__TEST_STATIC = True
from constants import *
from functions import load_data
import matplotlib.pyplot as plt
total_df = load_data(ALL_DAYS[0:5],
ignore_auctions=True,
correct_zero_volumes=True,
num_minutes_interval=1)
symbol = 'MMM'
sample_day = total_df[(total_df.Day == '20120924')
& (total_df.Symbol == symbol)]
fitter = VolumeEstimatorStatic(test=True)
model_fit_parameters = fitter.fit(total_df)
M_t = model_fit_parameters['M_t']
# plot
plt.plot([0.] + list(
np.cumsum(sample_day.Volume.values) / float(sample_day.Volume.sum())),
label='market')
#plt.plot(model_fit_parameters['inverse_solution'], label='inverse_solution')
from numba import prange #parallise loop
from generator import batch_generator_SRCNN
from functions import load_data, preprocess_data, cPSNR_callback, compute_steps
K.clear_session()
from tensorflow.compat.v1 import ConfigProto
from tensorflow.compat.v1 import InteractiveSession
config = ConfigProto()
config.gpu_options.allow_growth = True
session = InteractiveSession(config=config)
from sklearn.model_selection import train_test_split
from model import FSRCNN, FSRCNN_, SRCNN, SRCNNex, SRCNNv2, SRVGG16, FSRCNNv2, SRCNNv3, SRResnet
from model import custom_loss, PSNR, MSE
# load data
data_test = load_data("data/test.txt")
datas = load_data("data/train.txt")
data_train, data_val = train_test_split(datas,
test_size=0.1,
shuffle=True,
random_state=42)
## preprocess data
k = 5
scale = 3
resize = True
with_clearance = False
type_clearance = "sum"
version = 4
version_val = 1 if version != 4 else 4
from functions import load_data, train, evaluate, plot_result
from keras.models import Sequential
from keras.layers import Dense, Dropout, LeakyReLU
(x_train, y_train), (x_test, y_test) = load_data()
model = Sequential()
model.add(Dense(784, activation='relu', input_shape=x_train.shape[1:]))
model.add(Dense(128))
model.add(LeakyReLU(alpha=0.01))
model.add(Dense(64))
model.add(LeakyReLU(alpha=0.01))
model.add(Dropout(rate=0.25))
model.add(Dense(128))
model.add(LeakyReLU(alpha=0.01))
model.add(Dense(64))
model.add(LeakyReLU(alpha=0.01))
model.add(Dense(10, activation='softmax'))
model.compile(
loss='categorical_crossentropy',
optimizer='sgd',
metrics=['accuracy']
)
history = train(model, x_train, y_train)
plot_result(history)
evaluate(model, x_test, y_test)
class_num = 102
epochs = args.epochs
steps = 0
train_loss = 0
print_every = 30
data_dir = Path(args.data_dir)
train_dir = str(data_dir / 'train')
valid_dir = str(data_dir / 'valid')
test_dir = str(data_dir / 'test')
save_dir = str(Path(args.save_dir) / args.chpt_fn)
# load data
trainloader, testloader, validloader, train_data = functions.load_data(
train_dir, valid_dir, test_dir)
# load the model from torchvision
model = getattr(models, arch)(pretrained=True)
# Freeze parameters so we don't backprop through them
for param in model.parameters():
param.requires_grad = False
# Define our new classifier
classifier = nn.Sequential(nn.Linear(25088, hidden_units),
nn.Dropout(p=dropout_p), nn.ReLU(),
nn.Linear(hidden_units, class_num),
nn.LogSoftmax(dim=1))
model.classifier = classifier
NUM_CLIENTS = 4
NUM_EPOCHS = 100
BATCH_SIZE = 32
SHUFFLE_BUFFER = 500
NUM_AVERAGE_ROUND = 10
# Check the environment
warnings.simplefilter('ignore')
np.random.seed(0)
if six.PY3:
tff.framework.set_default_executor(tff.framework.create_local_executor())
tff.federated_computation(
lambda: 'The tensorflow federated environment is correctly setup!')()
# Load the data
emnist_train, emnist_test = load_data(path)
# Generate sample batch
example_dataset = emnist_train.create_tf_dataset_for_client(
emnist_train.client_ids[1])
example_element = iter(example_dataset).next()
preprocessed_example_dataset = preprocess(example_dataset, NUM_EPOCHS,
SHUFFLE_BUFFER, BATCH_SIZE)
sample_batch = tf.nest.map_structure(lambda x: x.numpy(),
iter(preprocessed_example_dataset).next())
# Create federated data for each client
sample_clients = emnist_train.client_ids[0:NUM_CLIENTS]
federated_train_data = make_federated_data(emnist_train, sample_clients,
NUM_EPOCHS, SHUFFLE_BUFFER,
BATCH_SIZE)
import torch
from model import Net
from functions import load_data, train_model, test_model, print_prediction
lr = 0.001
if __name__ == '__main__':
train, validation = load_data()
cuda = torch.cuda.is_available()
device = torch.device("cuda" if cuda else "cpu")
model = Net().to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
# train the model
train_model(train, optimizer, model, device)
# test the model
test_model(validation, model, device)
print_prediction(model, device)
