def CalculateBitFlips(): X_adv, _ = LoadData(DATA_PATH, file="npy") _, (X_test, _) = LoadData(DATA_PATH) lqvae = LQVAE(batch_size=BATCH_SIZE, max_iters=MAX_ITERS_LQVAE, latent_dim=LATENT_DIM, learnrate_init=LEARN_RATE_INIT_LQVAE, log_every=LOG_EVERY, save_every=SAVE_EVERY, image_path=IMAGE_PATH) classifier = Classifier(batch_size=BATCH_SIZE, max_iters=MAX_ITERS_CLASSIFIER, learnrate_init=LEARN_RATE_INIT_CLASSIFIER, log_every=LOG_EVERY, test_every=TEST_EVERY) lqvae.build_model_lqvae() classifier.build_model_classifier() saver = tf.train.Saver() with tf.Session() as session: saver.restore(session, MODEL_PATH) Bits(lqvae, classifier, X_test, X_adv, session)
def DriveFGSM(): (X_train, y_train), (X_test, y_test) = LoadData(DATA_PATH) lqvae = LQVAE(batch_size=BATCH_SIZE, max_iters=MAX_ITERS_LQVAE, latent_dim=LATENT_DIM, learnrate_init=LEARN_RATE_INIT_LQVAE, log_every=LOG_EVERY, save_every=SAVE_EVERY, image_path=IMAGE_PATH) classifier = Classifier(batch_size=BATCH_SIZE, max_iters=MAX_ITERS_CLASSIFIER, learnrate_init=LEARN_RATE_INIT_CLASSIFIER, log_every=LOG_EVERY, test_every=TEST_EVERY) lqvae.build_model_lqvae() classifier.build_model_classifier() saver = tf.train.Saver() with tf.Session() as session: saver.restore(session, MODEL_PATH) FGSM(lqvae, classifier, X_test, y_test, session, FGSM_IMAGE_PATH)
def EvaluatePerformance(): # _, (X, y) = LoadData(DATA_PATH) X, y = LoadData(DATA_PATH, file="npy") lqvae = LQVAE(batch_size=BATCH_SIZE, max_iters=MAX_ITERS_LQVAE, latent_dim=LATENT_DIM, learnrate_init=LEARN_RATE_INIT_LQVAE, log_every=LOG_EVERY, save_every=SAVE_EVERY, image_path=IMAGE_PATH) classifier = Classifier(batch_size=BATCH_SIZE, max_iters=MAX_ITERS_CLASSIFIER, learnrate_init=LEARN_RATE_INIT_CLASSIFIER, log_every=LOG_EVERY, test_every=TEST_EVERY) lqvae.build_model_lqvae() classifier.build_model_classifier() saver = tf.train.Saver() with tf.Session() as session: saver.restore(session, MODEL_PATH) Evaluate(lqvae, classifier, X, y, session)
def main(args): nlp = spacy.load('en_coref_lg') data_loader = LoadData(args.file) data = data_loader.get_tuples() for sample in data: doc = nlp(sample.text) corefs = unpack_spacy_doc(doc) if args.verbose: logger.info(corefs)
from matplotlib.colors import ListedColormap # Import lib files envs = load_dotenv(find_dotenv()) file = os.getenv("lib") sys.path.insert(0, file) from utils import LoadData from preprocessing import PreProcessing from visuals import ClassifierVisual # Import model library from sklearn.svm import SVC from sklearn.metrics import confusion_matrix # Import data dataset = LoadData("Social_Network_Ads.csv").data # Split the dataset X = dataset.iloc[:, [2,3]].values y = dataset.iloc[:, 4].values # Lets do some preprocessing... processor = PreProcessing() # Split the data X_train, X_test, y_train, y_test = processor.split(X, y, test_size=0.25) # scale the data X_train = processor.fit_scaler(X_train) X_test = processor.scale(X_test) # Lets fit the model now classifier = SVC(kernel='rbf', random_state=0)
type=int, default=50, help='number of classes used') parser.add_argument('--num_samples_train', type=int, default=15, help='number of samples per class used for training') parser.add_argument('--num_samples_test', type=int, default=5, help='number of samples per class used for testing') parser.add_argument('--seed', type=int, default=1, help='random seed') args = parser.parse_args() train_image, train_label, test_image, test_label = LoadData( args.num_classes, args.num_samples_train, args.num_samples_test, args.seed) train_loader = torch.utils.data.DataLoader(HWDataset( train_image, train_label), shuffle=True, batch_size=8) test_loader = torch.utils.data.DataLoader(HWDataset( test_image, test_label), shuffle=False, batch_size=1) model = HWModel2(args.num_classes) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model.to(device) optimizer = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.5) epoch_num = 500
# Import lib files envs = load_dotenv(find_dotenv()) file = os.getenv("lib") sys.path.insert(0, file) from utils import LoadData from preprocessing import PreProcessing from visuals import ClassifierVisual # Import model library from sklearn.linear_model import LogisticRegression from sklearn.metrics import confusion_matrix from sklearn.decomposition import PCA # Import data dataset = LoadData("Wine.csv").data # Split the dataset X = dataset.iloc[:, 0:13].values y = dataset.iloc[:, 13].values # Lets do some preprocessing... processor = PreProcessing() # Split the data X_train, X_test, y_train, y_test = processor.split(X, y, test_size=0.2) # scale the data X_train = processor.fit_scaler(X_train) X_test = processor.scale(X_test) # Apply PCA pca = PCA(n_components = 2)
# Import Requierd libraries from xgboost import XGBClassifier from sklearn.metrics import confusion_matrix from sklearn.model_selection import cross_val_score # Import lib files envs = load_dotenv(find_dotenv()) file = os.getenv("lib") sys.path.insert(0, file) from utils import LoadData from preprocessing import PreProcessing from visuals import ClassifierVisual # Import data dataset = LoadData("Churn_Modelling.csv").data X = dataset.iloc[:, 3:13].values y = dataset.iloc[:, 13].values # Lets do some preprocessing... processor = PreProcessing() # Encode the data (Country/Gender) X[:, 1] = processor.encode(X[:, 1]) X[:, 2] = processor.encode(X[:, 2]) X = processor.hot_encoding(data=X, features=[1]) X = X[:, 1:] # Split the data into training+test X_train, X_test, y_train, y_test = processor.split(X, y, test_size=0.2) # Fitting XGboost
from matplotlib.colors import ListedColormap import random import math import re from sklearn.feature_extraction.text import CountVectorizer from sklearn.naive_bayes import GaussianNB # Import lib files envs = load_dotenv(find_dotenv()) file = os.getenv("lib") sys.path.insert(0, file) from utils import LoadData from preprocessing import PreProcessing from visuals import ClassifierVisual from sklearn.metrics import confusion_matrix # Import Data dataset = LoadData("Restaurant_Reviews.tsv", seperator='\t').data ### Cleaning the texts ### ps = PorterStemmer() corpus = [] for i in range(0, dataset.shape[0]): review = dataset['Review'][i] review = re.sub(pattern='[^a-zA-Z\s]', repl='', string=review) review = review.lower() review = review.split() review = [ ps.stem(word) for word in review if not word in stopwords.words('english') ] review = " ".join(review) corpus.append(review)
def main(): samples = LoadData(data_path).get_tuples() print(samples)
import pandas as pd from sklearn.linear_model import LinearRegression import matplotlib.pyplot as plt from sklearn.svm import SVR from sklearn.ensemble import RandomForestRegressor from sklearn.linear_model import LinearRegression # Import lib files envs = load_dotenv(find_dotenv()) file = os.getenv("lib") sys.path.insert(0, file) from utils import LoadData from preprocessing import PreProcessing # Load data dataset = LoadData("Position_Salaries.csv").data # Split the dataset X = dataset.iloc[:, 1:2].values y = dataset.iloc[:, 2].values # Decision Tree Regression regressor_10 = RandomForestRegressor(n_estimators=10, random_state=0) regressor_10.fit(X, y) regressor_50 = RandomForestRegressor(n_estimators=50, random_state=0) regressor_50.fit(X, y) regressor_100 = RandomForestRegressor(n_estimators=100, random_state=0) regressor_100.fit(X, y) lin_reg = LinearRegression() lin_reg.fit(X, y)
import pandas as pd import matplotlib.pyplot as plt from matplotlib.colors import ListedColormap import random import math # Import lib files envs = load_dotenv(find_dotenv()) file = os.getenv("lib") sys.path.insert(0, file) from utils import LoadData from preprocessing import PreProcessing from visuals import ClassifierVisual # Import Data dataset = LoadData("Ads_CTR_Optimisation.csv").data # Implementing Random Selection N = 10000 d = 10 ads_selected = [] random_total_reward = 0 for n in range(0, N): ad = random.randrange(d) ads_selected.append(ad) reward = dataset.values[n, ad] random_total_reward = random_total_reward + reward random_total_reward #Visualize the random plt.hist(ads_selected)
np.set_printoptions(suppress=True) import pandas as pd from sklearn.model_selection import train_test_split from sklearn.linear_model import LinearRegression import matplotlib.pyplot as plt import statsmodels.formula.api as sm # Import lib files envs = load_dotenv(find_dotenv()) file = os.getenv("lib") sys.path.insert(0, file) from utils import LoadData from preprocessing import PreProcessing # Load data dataset = LoadData("50_Startups.csv").data # Split the dataset X = dataset.iloc[:, :-1].values y = dataset.iloc[:, 4].values # Using the PreProcessing class from preprocessing processor = PreProcessing() # Encoding dummy variables X = processor.dummy_encoding(data=X, feature_position=3) # Avoiding the dummy variable trap X = X[:, 1:] # Building the optimal model using Backward Elimination X = np.append(arr=np.ones((X.shape[0], 1)).astype(int), values=X, axis=1)
from matplotlib.colors import ListedColormap # Import lib files envs = load_dotenv(find_dotenv(usecwd=True)) file = os.getenv("lib") sys.path.insert(0, file) print(file) from utils import LoadData from preprocessing import PreProcessing from visuals import ClassifierVisual # Import the apriori classifier from apyori import apriori # Data Preprocessing dataset = LoadData("Market_Basket_Optimisation.csv", header=None).data # Get transactions transactions = [] for i in range(0, dataset.shape[0]): transactions.append( [str(dataset.values[i, j]) for j in range(0, dataset.shape[1])]) # create the model rules = apriori(transactions, min_support=0.003, min_confidence=0.2, min_lift=3, min_length=2) # Visualize!
from dotenv import load_dotenv, find_dotenv import numpy as np import pandas as pd from sklearn.linear_model import LinearRegression import matplotlib.pyplot as plt np.set_printoptions(suppress=True) # Import lib files envs = load_dotenv(find_dotenv()) file = os.getenv("lib") sys.path.insert(0, file) from utils import LoadData from preprocessing import PreProcessing # Load the data dataset = LoadData("Salary_Data.csv").data # Split the dataset X = dataset.iloc[:, :-1].values y = dataset.iloc[:, 1].values # Using the PreProcessing class from preprocessing processor = PreProcessing() # Split the data X_train, X_test, y_train, y_test = processor.split(X, y, test_size=0.2, random_state=0) # Fit Simple Linear Regression model regressor = LinearRegression() regressor.fit(X_train, y_train)
from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, LearningRateScheduler, TensorBoard from model import model_cnn, model_resnet from utils import LoadData, PARAM arg = sys.argv print(len(arg)) if len(arg) < 3: print("Usage: python3 <train_data_path> <output_model_path>") exit() train_path = arg[1] outfile = arg[2] ## preprocess data ## cross validation X_train, Y_train = LoadData(train_path) validation_rate = 0.1 index = np.int(X_train.shape[0] * (1 - validation_rate)) trainX, testX, trainY, testY = X_train[:index], X_train[ index:], Y_train[:index], Y_train[index:] ## data augmentation train_datagen = ImageDataGenerator(featurewise_center=True, featurewise_std_normalization=True, rotation_range=20, width_shift_range=0.2, height_shift_range=0.2, shear_range=0.2, zoom_range=0.2, horizontal_flip=False, vertical_flip=True)
import matplotlib.pyplot as plt from matplotlib.colors import ListedColormap # Import machine learning package import scipy.cluster.hierarchy as sch from sklearn.cluster import AgglomerativeClustering # Import lib files envs = load_dotenv(find_dotenv()) file = os.getenv("lib") sys.path.insert(0, file) from utils import LoadData from preprocessing import PreProcessing # Importing the dataset dataset = LoadData('Mall_Customers.csv').data X = dataset.iloc[:, [3, 4]].values # use dendogram to clusters dendrogram = sch.dendrogram(sch.linkage(X, method='ward')) plt.xlabel('Euclidean Distance') plt.ylabel('Customers') plt.title('Dendrogram') plt.show() # Fitting kmeans hc = AgglomerativeClustering(n_clusters=5, affinity='euclidean', linkage='ward') y_hc = hc.fit_predict(X)
from sklearn.impute import SimpleImputer from sklearn.preprocessing import LabelEncoder, OneHotEncoder, StandardScaler import numpy as np np.set_printoptions(suppress=True) import pandas as pd from sklearn.model_selection import train_test_split # Import lib files envs = load_dotenv(find_dotenv()) file = os.getenv("lib") sys.path.insert(0, file) from utils import LoadData from preprocessing import PreProcessing # Load the data dataset = LoadData("Data.csv").data # Split the dataset X = dataset.iloc[:, :-1].values y = dataset.iloc[:, 3].values # Handling missing data imputer = SimpleImputer(missing_values=np.nan, strategy='mean') imputer = imputer.fit(X[:, 1:3]) X[:, 1:3] = imputer.transform(X[:, 1:3]) X[:, 1:3] # Handling Categorical Variables encoder_X = LabelEncoder() one_hot_encoder = OneHotEncoder(categorical_features=[0]) X[:, 0] = encoder_X.fit_transform(X[:, 0])