Ejemplo n.º 1
0
def load_data_callback(country_in):

    app.server.logger.info('load_data_callback')

    if not country_in:
        raise PreventUpdate

    load_data(country_in, all_data_dict, app)
    region_drp, province_drp = data2dropdown(country_in, all_data_dict)
    return region_drp, province_drp, 'done'
Ejemplo n.º 2
0
    def run_model(self, datafile):
        """Run the prep-fews model

        Returns a dict witht the different performance measures
        """
        time_series = fun.load_data(datafile)

        for event in time_series['Runoff']:
            # calculate losses if no forecasting system is operational
            climate_loss = self.estimate_damage(event)

            # calculate losses if a forecasting system is operational
            forecast = self.issue_forecast(event)
            warning_outcome = self.assess_warning_outcome(event, forecast)
            warning_damage = self.get_warning_damage(event, warning_outcome,
                                                     self.preparedness)
            warning_loss = self.get_warning_loss(event, warning_outcome,
                                                 self.preparedness)

            # keep track of the states of the model
            self.forecast_ls.append(forecast)
            self.outcome_ls.append(warning_outcome)
            self.warning_loss_ls.append(warning_loss)
            self.climate_loss_ls.append(climate_loss)
            self.warning_damage_ls.append(warning_damage)
            self.preparedness_ls.append(self.preparedness)

            # update the variables
            if self.ct_prep is False:
                self.preparedness = self.update_preparedness(
                    warning_damage, self.preparedness)
Ejemplo n.º 3
0
def data() -> list:
    name_file = input("Wprowadź nazwę pliku: ")
    m = functions.load_data(name_file)
    print("Twoja macierz:")
    for s in m:
        print("wybór", *s)
    return m
Ejemplo n.º 4
0
def main():
    start_time = time.time()

    in_arg = args_input()
    # get the data
    dataset, dataloaders = functions.load_data()

    # get the model
    model = functions.get_model(in_arg.arch)

    # get the classifier, criterion, optimizer, device
    model, classifier, criterion, optimizer, device = network_param(
        model, in_arg.arch, in_arg.hidden_units, in_arg.learning_rate,
        in_arg.gpu)

    model.to(device)

    print('Training model...')
    functions.train_model(model, criterion, optimizer, device,
                          dataloaders['train'], dataloaders['val'],
                          in_arg.epochs)

    # validation on test data
    print('\nGetting results on test data accuracy...')
    functions.test_model(model, criterion, device, dataloaders['test'])

    # saving checkpoint on trained model
    print('\nSaving checkpoint for current trained model...')
    functions.save_checkpoint(model, optimizer, dataset, in_arg.arch,
                              in_arg.epochs, in_arg.save_dir)
    print('Checkpoint saved!')
Ejemplo n.º 5
0
 def __init__(self, name):
     print(name)
     self.name = name
     self.scores, self.best_scores, self.mean_scores, self.std_scores = functions.load_data(self.name) 
     self.N_population = 2
     self.last_gen = []
     for i in range(self.N_population):
         self.last_gen.append(torch.load(self.name+"/TorchSaves/Last_Gen"+str(998+i)+".pt"))
Ejemplo n.º 6
0
def data():
    name_file = input("Wprowadź nazwę pliku: ")
    m = functions.load_data(name_file)
    print("Twoja macierz:")
    print("A/B")
    for s in m:
        print(*s)
    return m
Ejemplo n.º 7
0
def main():
    data, optimal = f.load_data()
    params = f.load_parameters()
    print("Optimal:: {}\n".format(optimal))
    der_uebermensch, fitness_dictionary, error = genetik.NaturalSelection(
        len(data), params, data, optimal)
    print(der_uebermensch)

    if params["graph_data"] is True:
        f.show_graph(fitness_dictionary, params, error)
    else:
        quit()
Ejemplo n.º 8
0
    def __removing_from_blacklist(self):
        """Удаление пользователя с чёрного списка"""
        def remove_user(uid, black_list):
            """Удаление пользователя"""
            if uid in black_list:
                black_list.remove(uid)

                # уведомление
                message = "\n - Пользователь успешно удалён!"
                print(message + "\n")

                # логирование
                functions.logging('{} Удалён (ID): {}'.format(message, uid),
                                  'Отсутствует', 'Успешное удаление пользователя с чёрного листа')

            else:
                print("\n - Пользователь с данным ID не найден!\n")
            time.sleep(3)

        def get_id_from_console():
            """Получение ID пользователя с консоли, который будет удалён """
            try:
                got_id = int(input("\nВведите ID пользователя, которого хотите удалить: "))
                if len(str(got_id)) == 9:
                    return got_id
                else:
                    print("\n - ID должен состоять из 9 чисел")
                    return None
            except ValueError:
                print("\n - ID должен состоять только из чисел")

        while True:
            try:
                blacklist = functions.load_data(self.BLACKLIST)
            except FileNotFoundError as error:
                message = "\n - Файл не найден!"
                print(message + "\n")

                functions.logging(message, error)
                time.sleep(3)
            else:
                user_id = get_id_from_console()
                if user_id:

                    # удаление пользователя
                    remove_user(user_id, blacklist)

                    # обновление чёрного списка
                    self.__registration('w', self.BLACKLIST, blacklist)

                    break
Ejemplo n.º 9
0
    def __init__(self, df = None, file_id = "", strategy = "momentum_trading"):
        if df is None: 
            self.df = f.load_data()
        else: 
            self.df = df

        self.file = file_id
        self.df_rt = f.make_returns(self.df)
        self.df_lrt = f.make_log_returns(self.df)
        self.df_lrt_30ma = f.get_moving_average(self.df_lrt)
        self.indices = self.df_lrt.index
        self.n = len(self.df_lrt)
        self.tradecosts = 0.001
        self.strategy = strategy
        self.assets = self.df.columns
        self.endowment = np.random.choice(self.assets, 5, replace = False)
        self.startperiod = 31
        self.cur_date_ind = self.startperiod
        self.start_date = self.indices[self.cur_date_ind] 
        self.current_date = self.start_date
        self.cur_return = 0
        self.baseline_return = 0
        self.tradefreq = 14
        self.cash = 0
        self.cash_baseline = 0
        self.finish = False

        self.endowment = f.get_worst_performers(self.df_lrt_30ma, self.current_date)
        self.cash -= np.sum(self.df.loc[self.current_date, self.endowment] * (1 + self.tradecosts))
        self.cash_baseline -= np.sum(self.df.loc[self.current_date, :]) * (1 + self.tradecosts)

        ## current value when cashing out
        self.cur_value = self.cash + np.sum(self.df.loc[self.current_date, self.endowment]) * (1 - self.tradecosts)
        self.cur_value_baseline = self.cash_baseline + np.sum(self.df.loc[self.current_date, :]) * (1 - self.tradecosts)
        self.overall_return = 1
        self.overall_return_base = 1
        self.results = []
        self.threshold = None
Ejemplo n.º 10
0
def main_definition():
    """Определение управляющего бота, регистрации, автоответчика"""
    try:
        configs = functions.load_data('data/config-bot.json')
        bot_token, my_id = configs['TOKEN'], configs['my_ID']

    except FileNotFoundError as error:
        message = "\n - Файл с конфигурациями для бота не найден! Зарегестрируйте данные!"
        print(message + "\n")

        functions.logging(message, error)
        functions.do_command()
    else:
        bot = TeleBot(bot_token)
        bot.remove_webhook()

        registration = Register()
        answering_bot = AnsweringBot()

        if not os.path.exists(registration.LOGS):
            functions.creating_file(registration.LOGS)

        return bot, registration, answering_bot, my_id
Ejemplo n.º 11
0
import psycopg2
import seaborn as sns
from sklearn.feature_extraction.text import CountVectorizer
sklearn.feature_extraction import text

## TODO: get rid of:
import common

import functions as f


matplotlib.style.use('ggplot')


## TODO: fix unicode here.
df = f.load_data()


# ## Examining and cleaning data


# Extract features
# TODO: remove stopwords before vectorizing, because the stop words don't capture bigrams.

## this is for 

# In[14]:


## this model, with 40 topics, is not bad.
m = lda.LDA(n_topics=40, random_state=0, n_iter=200)
Ejemplo n.º 12
0
			#print 'no data willbe plotted...'
			plotdata=dd
		elif (dd=='yes') or (args.VAL==[]):
			#print 'plot everything...'
			plotdata='yes'
		else:
			sys.exit("Error (-plotdata): Only 'yes' or 'no' values are admisible.")
	#######################
	### MAIN ##############
	#######################
	sys.stdout.write("\a")
	#loading data points ('N' points for each one of the 'DIM' CVs)
	N=file_lines(args.INPUT)
	DIM=file_cols(args.INPUT)

	P1=load_data(args.INPUT,N,DIM)
	dK1=[]
	dK2=[]
	conv_all=[]

	# estimate the initial string (P2) of size Min
	P2 = string_ini(P1,DIM,args.K)
	Pini=string_ini(P1,DIM,args.K)
	PiniT=zip(*Pini)

	# maximun distances of the data to the images of the string
	maxdistances=maxvec(P1,P2)

	# Estimate the mean of the voronoid cells
	P3, fail, free_energy, allpoints, closeP, closeI = voroid_mean(P1,P2,DIM,maxdistances,0)
Ejemplo n.º 13
0
    def __init__(self, nb_epochs, initial_lr):
        self.epochs = nb_epochs
        self.initial_lr = initial_lr

    def __call__(self, epoch_idx):
        if epoch_idx < self.epochs * 0.25:
            return self.initial_lr
        elif epoch_idx < self.epochs * 0.50:
            return self.initial_lr * 0.2
        elif epoch_idx < self.epochs * 0.75:
            return self.initial_lr * 0.04
        return self.initial_lr * 0.008


# Load Base
image, gender, age = ft.load_data("serializer/MATimdb64.mat")

x_data = image
y_data_g = np_utils.to_categorical(gender, 2)
y_data_a = np_utils.to_categorical(age, 101)

# Fatores importantes:
# optimizers, largura da rede, profundidade da rede, losses, val_split,

# model WidesResNet (WRN) dupla saida(age,gender)
model = WideResNet(64, depth=16, k=8)()
#model.summary()

#optimizers
model.compile(SGD(lr=0.1, momentum=0.9, nesterov=True),
              loss=['categorical_crossentropy', 'categorical_crossentropy'],
from functions import get_best_wide, load_results, aggregate_results, load_data, resample_ohlc, hma_calc_old
import math

pair = 'TOMOBTC'
strat = 'hma_strat'
timescale = '1h'
train = 2000
test = 50
train_string = f'{train}-{test}'
params = 'lengths5-501-2'
metric = 'sqn'

print('- Loading data')
price, vol = load_data(pair)
print('- Resampling data')
price, vol = resample_ohlc(price, vol, timescale)
print(f'len(price): {len(price)}')

print('- Loading results')
df_dict = load_results(strat, pair, timescale, train_string, params)

print('- Running get_best')
best = get_best_wide(metric, df_dict, train, test)

print('Running hma_calc')
hma_25 = hma_calc_old(price, 25)
print('Running hma_calc')
hma_327 = hma_calc_old(price, 327)

nan_count = 0
for q in range(len(hma_25)):
def SM_test(summaryMeasure,
            k,
            data,
            label,
            data_test,
            label_test,
            batch_size,
            is_mmens=False):

    train_split_k = train_split[k]
    val_split_k = val_split[k]
    #reset the model before new cv
    model = CNN_model()
    model.to(device)
    # Loss and optimizer
    criterion = nn.BCEWithLogitsLoss()
    optimizer = optim.SGD(model.parameters(),
                          momentum=0.9,
                          lr=0.001,
                          weight_decay=1e-3)

    # load data
    train_split_k = train_split[k]
    testData = load_data(
        split='test',
        train_split=train_split_k,
        val_split=val_split_k,
        summaryMeasure=summaryMeasure,
        data=data,
        label=label,  # need train data for mean and sd normalization
        data_test=data_test,
        label_test=label_test)

    test_loader = DataLoader(dataset=testData,
                             batch_size=batch_size,
                             shuffle=True)

    # set and load best model
    model = CNN_model()
    model.load_state_dict(
        torch.load(MODEL_STORE_PATH + '/' + summaryMeasure + '_BEST_model' +
                   '_CV' + str(k + 1) + '.pt'))

    model.to(device)
    # START TRAINING FOR THIS CV
    # TEST EVALUATION
    model.eval()

    total_ep = 0.0
    correct_ep = 0.0
    loss_ep = 0.0
    current_ep_acc = 0.0
    current_ep_loss = 0.0
    total_step_v = len(test_loader)

    # F1 calc
    F1_labels = []
    F1_pred = []

    for i, (images, labels
            ) in enumerate(test_loader):  #TEST THE MODEL ON THE VALIDATION

        if device.type == "cuda":
            images = images.to(device)
            labels = labels.to(device)
        labels = labels.reshape(len(labels), 1).type(torch.cuda.FloatTensor)

        outputs_v = model(images)  # the forward uses the entire batch together
        predicted = outputs_v.data > 0.0  #define True if >0.5 and False if <0.5

        # collect true labels and predicted for metrix calc

        if i == 0:
            F1_labels = labels.int().cpu().numpy()
            F1_pred = predicted.int().cpu().numpy()
        else:
            F1_labels = np.concatenate((F1_labels, labels.int().cpu().numpy()))
            F1_pred = np.concatenate((F1_pred, predicted.int().cpu().numpy()))

        loss_v = criterion(outputs_v, labels)

        total_v = labels.size(
            0)  # get it in case the last batch has different number of sample
        correct_v = (predicted == labels
                     ).sum().item()  # sum the correct answers for this batch
        #acc_list_v.append(correct_v / total_v) # n of correct / n of sample in this batch

        total_ep += total_v  # sum all the samples. it must end up being the tot training set
        correct_ep += correct_v  # sum together the right answers in entire training set
        loss_ep += (loss_v.item() * len(labels))

        current_ep_acc = (correct_ep / total_ep) * 100
        current_ep_loss = loss_ep / total_ep

    acc = accuracy_score(F1_labels, F1_pred)
    bal_acc = balanced_accuracy_score(F1_labels, F1_pred)
    F1_Score = f1_score(F1_labels, F1_pred,
                        average='weighted')  #, zero_division=1)
    tn, fp, fn, tp = confusion_matrix(F1_labels, F1_pred).ravel()

    print('CV {}, sumMeasure: {} Test Accuracy of the model is: {}, F1: {}%'.
          format(k + 1, summaryMeasure, current_ep_acc, F1_Score))

    if is_mmens == True:
        return F1_labels.reshape(-1), F1_pred.reshape(-1)
    else:
        return bal_acc, f1_score
def SM_train_val_kfoldCV(summaryMeasure, data, label, batch_size, num_epochs,
                         n_split):

    # CROSSVALIDATION LOOP
    for k in np.arange(n_split):
        train_split_k = train_split[k]
        val_split_k = val_split[k]

        #reset the model before new cv
        model = CNN_model()
        model.to(device)
        # Loss and optimizer
        criterion = nn.BCEWithLogitsLoss()
        optimizer = optim.SGD(model.parameters(),
                              momentum=0.9,
                              lr=0.001,
                              weight_decay=1e-3)

        # load data
        trainData = load_data(split='train',
                              train_split=train_split_k,
                              val_split=val_split_k,
                              summaryMeasure=summaryMeasure,
                              data=data,
                              label=label)
        valData = load_data(split='val',
                            train_split=train_split_k,
                            val_split=val_split_k,
                            summaryMeasure=summaryMeasure,
                            data=data,
                            label=label)

        train_loader = DataLoader(dataset=trainData,
                                  batch_size=batch_size,
                                  shuffle=True)
        val_loader = DataLoader(dataset=valData,
                                batch_size=batch_size,
                                shuffle=True)

        # START TRAINING FOR THIS CV

        # collect loss and acc of each epoch
        epoch_loss_train = []
        epoch_acc_train = []

        epoch_loss_val = []
        epoch_balacc_val = []
        epoch_F1_Score = []
        best_acc = None

        # TRAIN THE MODEL
        for epoch in range(num_epochs):
            total_ep = 0.0
            correct_ep = 0.0
            loss_ep = 0.0

            model.train()

            for i, (images, labels) in enumerate(train_loader):

                if device.type == "cuda":
                    images = images.to(device)
                    labels = labels.to(device)
                labels = labels.reshape(len(labels),
                                        1).type(torch.cuda.FloatTensor)

                # Run the forward pass
                outputs = model(images)
                loss = criterion(outputs, labels)

                # Backprop and perform  optimisation
                optimizer.zero_grad()
                loss.backward()
                optimizer.step()

                # Track the accuracy
                total = labels.size(0)
                predicted = outputs.data > 0.0
                correct = (predicted == labels).sum().item()

                total_ep += total
                correct_ep += correct
                loss_ep += (loss.item() * len(labels))

            epoch_acc_train.append((correct_ep / total_ep))
            epoch_loss_train.append(loss_ep / total_ep)

            # Test the model on validation data
            model.eval()

            total_ep = 0.0
            correct_ep = 0.0
            loss_ep = 0.0

            # F1 calc
            F1_labels = []
            F1_pred = []

            for i, (images, labels) in enumerate(val_loader):

                if device.type == "cuda":
                    images = images.to(device)
                    labels = labels.to(device)
                labels = labels.reshape(len(labels),
                                        1).type(torch.cuda.FloatTensor)

                outputs_v = model(
                    images)  # the forward uses the entire batch together
                predicted = outputs_v.data > 0.0  #define True if >0.5 and False if <0.5

                # collect true labels and predicted for metrix calc
                if i == 0:
                    F1_labels = labels.int().cpu().numpy()
                    F1_pred = predicted.int().cpu().numpy()
                else:
                    F1_labels = np.concatenate(
                        (F1_labels, labels.int().cpu().numpy()))
                    F1_pred = np.concatenate(
                        (F1_pred, predicted.int().cpu().numpy()))

                loss_v = criterion(outputs_v, labels)

                total_v = labels.size(0)
                correct_v = (predicted == labels).sum().item()

                total_ep += total_v
                correct_ep += correct_v
                loss_ep += (loss_v.item() * len(labels))

            # calculate metrices of this epoch
            current_ep_loss = loss_ep / total_ep

            current_ep_acc = (correct_ep / total_ep)
            acc = accuracy_score(F1_labels, F1_pred)
            balacc = balanced_accuracy_score(F1_labels, F1_pred)

            F1_Score = f1_score(F1_labels, F1_pred, average='weighted')
            tn, fp, fn, tp = confusion_matrix(F1_labels, F1_pred).ravel()

            if not best_acc or best_acc < current_ep_acc:
                best_acc = current_ep_acc
                torch.save(
                    model.state_dict(),
                    MODEL_STORE_PATH + '/' + summaryMeasure + '_BEST_model' +
                    '_CV' + str(k + 1) + '.pt')

            # collect matrixes of all the epochs
            epoch_loss_val.append((loss_ep / total_ep))
            epoch_balacc_val.append(balacc)

        print('{}, CV_{}, best accuracy = {}'.format(summaryMeasure, k + 1,
                                                     acc))
Ejemplo n.º 17
0
# path to data that has been cropped into smaller same-sized chucks
cropped_ground_truth_path_train = '/run/media/henryp/HenryHDD/DataSets/CMU/Formatted/Train/Cropped_data/sliced_'

cropped_ground_truth_path_test = '/run/media/henryp/HenryHDD/DataSets/CMU/Formatted/Test/Cropped_data/'

if run_on == 'GPU':
    device = torch.device('cuda:0')
else:
    device = torch.device('cpu')

print('============')
print('Training on: ' + str(device))
print('============')

training_partition, training_set_size = load_data(
    ground_truth_dir=cropped_ground_truth_path_train, type='train')

training_set = DataSet(list_IDS=training_partition['train'],
                       data_dir=cropped_ground_truth_path_train,
                       clip_length=64)
training_generator = data.DataLoader(training_set, **params)

testing_partition, testing_set_size = load_data(
    ground_truth_dir=cropped_ground_truth_path_test, type='test')

testing_set = DataSet(list_IDS=testing_partition['test'],
                      data_dir=cropped_ground_truth_path_test,
                      clip_length=64)

testing_generator = data.DataLoader(testing_set, **params)
Ejemplo n.º 18
0
                    help='Turn GPU mode on or off, default set to "off".')

results = parser.parse_args()

data_dir = results.data_directory
save_dir = results.save_directory
learning_rate = results.l_rate
dropout_rate = results.dropout_rate
hidden_unit_count = results.hidden_units
epochs = results.num_epochs
gpu_mode = results.gpu

arch = results.pretrained_model

# Load and preprocess data
train_loader, test_loader, validate_loader, train_data, test_data, validate_data = load_data(
    data_dir)

# Load pretrained model
pre_train_model = results.pretrained_model
model = getattr(models, pre_train_model)(pretrained=True)

# Build and attach new classifier
input_unit_count = model.classifier[0].in_features
build_classifier(model, input_unit_count, hidden_unit_count, dropout_rate)

# Using a NLLLoss as output is LogSoftmax
criterion = nn.NLLLoss()

# Using Adam optimiser algorithm - uses concept of momentum to add fractions
# to previous gradient descents to minimize loss function
optimizer = optim.Adam(model.classifier.parameters(), learning_rate)
    # constants
    W = 20
    f = 1

    def usage():
        print "Usage:\n%s [cross_val|simulation]" % sys.argv[0]

    if len(sys.argv) < 2:
        usage()

    if sys.argv[1] == 'cross_val':

        CV_LEN = 10
        total_df = load_data(ALL_DAYS[:W + CV_LEN],
                             ignore_auctions=True,
                             correct_zero_volumes=True,
                             num_minutes_interval=1)

        lambdas = [np.inf]
        for b in range(1, 10):

            rolling_simulator(total_df,
                              lambdas,
                              W,
                              analyze_day,
                              volume_model_meta_params={
                                  'num_factors': f,
                                  'bandwidth': b
                              })

            total_df['DynamicSolinf_b_%d' % b] = total_df.DynamicSolinf
Ejemplo n.º 20
0
        return parser
#--------------------------------
_p    = _make_parser()
__doc__ += _p.format_help()

######################################################
if __name__ == '__main__':
        args = _p.parse_args()

	#######################################
	############ CHECK_PCURVE ################
	####################################### 
	#File with the pcurv estate (output from pcurve.py)
	## loading pcurve state 
	N, DIM, Min, Mout, smooth, Niter, Ps, Psout, converg, free_energy, allpoints, closeP, closeI, closePP, closeII = loadpcurve(args.file1)
	P1=load_data(args.file2,N,DIM)
	print '#######################################################'
	print '############# pcurve run parameters  ##################'
	print '#######################################################'
	print '                                               '
	print 'N      (Size of the CV vectors)            = ',N
	print 'DIM    (# of CVs)                          = ',DIM
	print 'Min    (#of points on the string)          = ',Min
	print 'Mout   (# of final reparametrized points)  = ',Mout
	print 'smooth (strength of the smoothing [0-1])   = ',smooth
	print 'Niter  (# of iterations)                   = ',Niter
	print '                                             '
	print '#######################################################'

	if args.choice==0:
		PPP=Ps
Ejemplo n.º 21
0
    def __removing_from_user_list(self):
        """Удаление пользователя со списка друзей"""
        def find_key_via_value(uid, got_list):
            """Поиск ключа с помощью значения, для получения пользователя"""
            if type(uid) == int:
                if uid in got_list.values():
                    i = 0
                    for value in got_list.values():
                        if value == uid:
                            return list(got_list.keys())[i]
                        else:
                            i += 1
                else:
                    return None
            else:
                return uid

        def get_user_from_console():
            """Получение данных (ID или Имя) пользователя, который будет удалён со списка друзей"""
            got_user = input("\nВедите ID или имя пользователя, которого хотите удалить: ")
            try:
                got_user = int(got_user)
                if len(str(got_user)) == 9:
                    return got_user
                else:
                    print("\n - ID должен состоять из 9 чисел")
                    return None
            except ValueError:
                return got_user

        def remove_user(user_id, users_list):
            person = find_key_via_value(user_id, users_list)
            if not person:
                print("\n - Пользователь с данным ID не найден!")
            else:
                if person.capitalize() in users_list.keys():
                    del users_list[person.capitalize()]

                    # уведомление
                    message = "\n - Пользователь успешно удалён!"
                    print(message + "\n")

                    # логирование
                    functions.logging('{} Удалён: {}'.format(message, person.capitalize()),
                                      'Отсутствует', 'Успешное удаление пользователя со списка друзей')
                else:
                    print("\n - Пользователь с данным Именем не найден!")
            time.sleep(3)

        while True:
            try:
                user_list = functions.load_data(self.USERS)
            except FileNotFoundError as error:
                message = "\n - Файл не найден"
                print(message + "\n")

                # логирование
                functions.logging(message, error)
                time.sleep(3)
            else:
                user = get_user_from_console()
                if user:
                    # удаление пользователя
                    remove_user(user, user_list)

                    # обновление списка друзей
                    self.__registration('w', self.USERS, user_list)
                    break
Ejemplo n.º 22
0
import functions as f
import pandas as pd
import numpy as np
from sklearn.preprocessing import MinMaxScaler
import matplotlib.pyplot as plt
import seaborn as sns
from pylab import rcParams
from sklearn.model_selection import train_test_split
from keras.models import load_model
import  tensorflow as tf
rcParams['figure.figsize'] = 12, 6
sns.set(style='whitegrid', palette='muted', font_scale=1.5)


path = "dados\justica_eleitoral.json"
dataLoad= f.load_data(path)
encoded_seqs = f.prepare_inputs(dataLoad)

autoencoder = load_model('dados\model_seqs2.h5')

# ### Carregando novamente a base de dados com as anomalias
scaler = MinMaxScaler()
scaled_seqs = scaler.fit_transform(encoded_seqs)

# fazer a predição com base na rede treinada
predicted = autoencoder.predict(scaled_seqs)

# MSE em termo de erro
mse = np.mean(np.power(scaled_seqs - predicted, 2), axis=1)

#Data Frame com MSE
Ejemplo n.º 23
0
st.markdown(intro_markdown, unsafe_allow_html=True)
st.markdown("---")

# Sidebar inputs
st.sidebar.header('k-means clustering')
st.sidebar.write(
    'Select a date range and the number of clusters, then click Run to see a preview of the clustered data'
)
start_date = st.sidebar.date_input('Start date', date(2018, 1, 1))
end_date = st.sidebar.date_input('End date', date.today())
slider_input = st.sidebar.slider('Number of clusters (k)',
                                 min_value=1,
                                 max_value=10)

# Load cleaned data
runs = load_data()

# k-means clustering
if st.sidebar.button('Run'):
    output = clustering(runs, start_date, end_date, slider_input)

    cluster_counts = output[0]
    st.sidebar.write('Clusters')
    st.sidebar.dataframe(cluster_counts)

    st.write('Preview')
    bucket_data = output[1]
    st.dataframe(bucket_data.head())
    st.sidebar.markdown(filedownload(bucket_data), unsafe_allow_html=True)
st.sidebar.markdown("---")
Ejemplo n.º 24
0
if gpu_mode:
    print('GPU mode is selected')
    if torch.cuda.is_available() is False:
        print('please check the hardware. No GPU is detected')
        while input("Do you want to continue? [y/n]") == "n":
            exit()
else:
    print('Program will train on CPU .. will take longer time')
    while input("Do you want to continue running on CPU mode? [y/n]") == "n":
        exit()
print('... dont switch off computer .. training..')

start = time()

# working on data .. loading processing.. converting it into tensor etc..
trainloader, testloader, validloader, train_data, test_data, valid_data = load_data(
    data_dir)

# loading a pretrained model
pre_tr_model = results.pretrained_model
model = getattr(models, pre_tr_model)(pretrained=True)

# building a new classifier and replacing it with old classifier
input_units = model.classifier[0].in_features
build_classifier(model, input_units, hidden_units, dropout)

# criterion.. NLLoss selected since we provided logsoftmax for classifier
criterion = nn.NLLLoss()
# optimizer
optimizer = optim.Adam(model.classifier.parameters(), learning_rate)

# train model
Ejemplo n.º 25
0
from functions import visualization_cluster
from functions import load_data
from sklearn.cluster import DBSCAN
from sklearn.manifold import Isomap

# load data
_, times, _, _, duration = load_data()

# convert feature times and duration into ms
duration = [d / 1e6 for d in duration]
times = [[t / 1e6 for t in sample] for sample in times]

# remove 0.14 and 4.22 are most likely to be measurment errors
for _ in range(2):
    index = duration.index(min(duration))
    del times[index]
    del duration[index]

model = DBSCAN(eps=25, min_samples=100, n_jobs=-1, algorithm='ball_tree')

# visualization with Isomap
viz_model = Isomap(n_neighbors=15, n_components=2, n_jobs=-1)
cluster = model.fit_predict(times)
visualization_cluster(times, cluster, duration, viz_model, 25000,
                      "features_duration_isomap")
Ejemplo n.º 26
0
#%%
import pandas as pd
import numpy as np
import fbprophet as prop
import functions
import matplotlib.pyplot as plt
import os
import csv
import glob
import datetime
import joblib
import time

#%%
#get our data
df_train, df_test = functions.load_data()

#%%
#create the group by object
groups = df_train.groupby(['location', 'payment_type'])
forecasts = pd.DataFrame()

print('processing ' + str(len(groups)) + ' partitions ')

#%%
s = time.time()

for group_name, g in groups:

    location = str(g['location'].unique())
    payment_type = str(g['payment_type'].unique())
Ejemplo n.º 27
0
                    dest="hidden_units",
                    default=256,
                    type=int)
parser.add_argument('--dropout', dest="dropout", default=0.2)
parser.add_argument('--learning_rate', dest="learning_rate", default=0.001)
parser.add_argument('--gpu', dest="gpu", default=True, type=bool)
parser.add_argument('--epochs', dest="epochs", default=2, type=int)
parser.add_argument('--print_every', dest="print_every", default=10, type=int)
parser.add_argument('--save_dir',
                    dest="save_dir",
                    default="./checkpoint.pth",
                    type=str)
parsed = parser.parse_args()
data_dir = parsed.data_dir
arch = parsed.arch
hidden_units = parsed.hidden_units
dropout = parsed.dropout
learning_rate = parsed.learning_rate
gpu = parsed.gpu
epochs = parsed.epochs
print_every = parsed.print_every
save_dir = parsed.save_dir

trainloader, validloader, class_to_idx = functions.load_data(data_dir)
model, criterion, optimizer = functions.setup(arch, hidden_units, dropout,
                                              learning_rate, gpu)
functions.train_network(model, criterion, optimizer, trainloader, validloader,
                        epochs, print_every, gpu)
functions.save_checkpoint(save_dir, arch, hidden_units, dropout, learning_rate,
                          gpu, model, optimizer, class_to_idx)
Ejemplo n.º 28
0
            result['Sigma'], result['S_orig'], result['U'] = \
                self._computeSigma()
        else:
            result['Sigma'] = self._computeSigma()
        return result


if __name__ == "__main__":
    __TEST_STATIC = True

    from constants import *
    from functions import load_data
    import matplotlib.pyplot as plt

    total_df = load_data(ALL_DAYS[0:5],
                         ignore_auctions=True,
                         correct_zero_volumes=True,
                         num_minutes_interval=1)

    symbol = 'MMM'
    sample_day = total_df[(total_df.Day == '20120924')
                          & (total_df.Symbol == symbol)]

    fitter = VolumeEstimatorStatic(test=True)
    model_fit_parameters = fitter.fit(total_df)
    M_t = model_fit_parameters['M_t']

    # plot
    plt.plot([0.] + list(
        np.cumsum(sample_day.Volume.values) / float(sample_day.Volume.sum())),
             label='market')
    #plt.plot(model_fit_parameters['inverse_solution'], label='inverse_solution')
Ejemplo n.º 29
0
from numba import prange  #parallise loop
from generator import batch_generator_SRCNN
from functions import load_data, preprocess_data, cPSNR_callback, compute_steps
K.clear_session()
from tensorflow.compat.v1 import ConfigProto
from tensorflow.compat.v1 import InteractiveSession
config = ConfigProto()
config.gpu_options.allow_growth = True
session = InteractiveSession(config=config)

from sklearn.model_selection import train_test_split
from model import FSRCNN, FSRCNN_, SRCNN, SRCNNex, SRCNNv2, SRVGG16, FSRCNNv2, SRCNNv3, SRResnet
from model import custom_loss, PSNR, MSE
# load data
data_test = load_data("data/test.txt")
datas = load_data("data/train.txt")

data_train, data_val = train_test_split(datas,
                                        test_size=0.1,
                                        shuffle=True,
                                        random_state=42)

## preprocess data
k = 5
scale = 3
resize = True
with_clearance = False
type_clearance = "sum"
version = 4
version_val = 1 if version != 4 else 4
Ejemplo n.º 30
0
from functions import load_data, train, evaluate, plot_result
from keras.models import Sequential
from keras.layers import Dense, Dropout, LeakyReLU

(x_train, y_train), (x_test, y_test) = load_data()

model = Sequential()
model.add(Dense(784, activation='relu', input_shape=x_train.shape[1:]))
model.add(Dense(128))
model.add(LeakyReLU(alpha=0.01))
model.add(Dense(64))
model.add(LeakyReLU(alpha=0.01))
model.add(Dropout(rate=0.25))
model.add(Dense(128))
model.add(LeakyReLU(alpha=0.01))
model.add(Dense(64))
model.add(LeakyReLU(alpha=0.01))
model.add(Dense(10, activation='softmax'))

model.compile(
    loss='categorical_crossentropy',
    optimizer='sgd', 
    metrics=['accuracy']
)

history = train(model, x_train, y_train)
plot_result(history)
evaluate(model, x_test, y_test)
Ejemplo n.º 31
0
    class_num = 102

    epochs = args.epochs
    steps = 0
    train_loss = 0
    print_every = 30

    data_dir = Path(args.data_dir)
    train_dir = str(data_dir / 'train')
    valid_dir = str(data_dir / 'valid')
    test_dir = str(data_dir / 'test')

    save_dir = str(Path(args.save_dir) / args.chpt_fn)

    # load data
    trainloader, testloader, validloader, train_data = functions.load_data(
        train_dir, valid_dir, test_dir)

    # load the model from torchvision
    model = getattr(models, arch)(pretrained=True)

    # Freeze parameters so we don't backprop through them
    for param in model.parameters():
        param.requires_grad = False

    # Define our new classifier
    classifier = nn.Sequential(nn.Linear(25088, hidden_units),
                               nn.Dropout(p=dropout_p), nn.ReLU(),
                               nn.Linear(hidden_units, class_num),
                               nn.LogSoftmax(dim=1))
    model.classifier = classifier
Ejemplo n.º 32
0
NUM_CLIENTS = 4
NUM_EPOCHS = 100
BATCH_SIZE = 32
SHUFFLE_BUFFER = 500
NUM_AVERAGE_ROUND = 10

# Check the environment
warnings.simplefilter('ignore')
np.random.seed(0)
if six.PY3:
    tff.framework.set_default_executor(tff.framework.create_local_executor())
tff.federated_computation(
    lambda: 'The tensorflow federated environment is correctly setup!')()

# Load the data
emnist_train, emnist_test = load_data(path)

# Generate sample batch
example_dataset = emnist_train.create_tf_dataset_for_client(
    emnist_train.client_ids[1])
example_element = iter(example_dataset).next()
preprocessed_example_dataset = preprocess(example_dataset, NUM_EPOCHS,
                                          SHUFFLE_BUFFER, BATCH_SIZE)
sample_batch = tf.nest.map_structure(lambda x: x.numpy(),
                                     iter(preprocessed_example_dataset).next())

# Create federated data for each client
sample_clients = emnist_train.client_ids[0:NUM_CLIENTS]
federated_train_data = make_federated_data(emnist_train, sample_clients,
                                           NUM_EPOCHS, SHUFFLE_BUFFER,
                                           BATCH_SIZE)
Ejemplo n.º 33
0
import torch
from model import Net
from functions import load_data, train_model, test_model, print_prediction

lr = 0.001

if __name__ == '__main__':
    train, validation = load_data()
    cuda = torch.cuda.is_available()
    device = torch.device("cuda" if cuda else "cpu")
    model = Net().to(device)
    optimizer = torch.optim.Adam(model.parameters(), lr=lr)

    # train the model
    train_model(train, optimizer, model, device)
    # test the model
    test_model(validation, model, device)

    print_prediction(model, device)