def main():
tf.reset_default_graph()
drive.mount('/content/gdrive')
x_train, y_train, x_test, y_test = read_data_chars()
# Create the model
x = tf.placeholder(tf.int64, [None, MAX_DOCUMENT_LENGTH])
y_ = tf.placeholder(tf.int64)
logits = rnn_model(x)
correct_prediction, accuracy, classification_errors = configure_statistics(logits, tf.one_hot(y_, MAX_LABEL))
entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=tf.one_hot(y_, MAX_LABEL), logits=logits))
minimizer = tf.train.AdamOptimizer(lr)
grads_and_vars = minimizer.compute_gradients(entropy)
# Gradient clipping
grad_clipping = tf.constant(2.0, name="grad_clipping")
clipped_grads_and_vars = []
for grad, var in grads_and_vars:
clipped_grad = tf.clip_by_value(grad, -grad_clipping, grad_clipping)
clipped_grads_and_vars.append((clipped_grad, var))
# Gradient updates
train_op = minimizer.apply_gradients(clipped_grads_and_vars)
N = len(x_train)
indexes = np.arange(N)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
# training
test_acc_pts = []
training_cost_pts = []
training_acc_pts = []
epoch_times = []
randomizedX, randomizedY = x_train,y_train
testX,testY = x_test,y_test
total_start = timer()
for e in range(epochs):
np.random.shuffle(indexes)
randomizedX, randomizedY = randomizedX[indexes], randomizedY[indexes]
experiment_start = timer()
for start, end in zip(range(0, N+1, batch_size), range(batch_size, N+1, batch_size)):
sess.run([train_op], {x: randomizedX[start:end], y_: randomizedY[start:end]})
experiment_end = timer()
#upon completing an epoch of training, collect required stats
loss_pt = entropy.eval(feed_dict={x: randomizedX, y_: randomizedY})
training_cost_pts.append(loss_pt)
test_acc_pt = accuracy.eval(feed_dict={x: x_test, y_: y_test})
test_acc_pts.append(test_acc_pt)
training_acc_pt = accuracy.eval(feed_dict={x: x_train, y_: y_train})
training_acc_pts.append(training_acc_pt)
epoch_times.append(experiment_end-experiment_start)
if(e % 100 == 0):
print('epoch', e, 'entropy', loss_pt, 'time', experiment_end - experiment_start)
total_end = timer()
print('Time taken', total_end - total_start)
np_test_accs = np.array(test_acc_pts)
np_test_accs = np.expand_dims(np_test_accs,axis=0)
np_training_accs = np.array(training_acc_pts)
np_training_accs = np.expand_dims(np_training_accs,axis=0)
np_training_costs = np.array(training_cost_pts)
np_training_costs = np.expand_dims(np_training_costs,axis=0)
np_times = np.expand_dims((total_end - total_start,np.mean(epoch_times)),axis=0)
np.savetxt('/content/gdrive/My Drive/Colab Notebooks/6c_char_clip_training_cost.txt',np_training_costs)
np.savetxt('/content/gdrive/My Drive/Colab Notebooks/6c_char_clip_test_accs.txt',np_test_accs)
np.savetxt('/content/gdrive/My Drive/Colab Notebooks/6c_char_clip_training_acc.txt',np_training_accs)
np.savetxt('/content/gdrive/My Drive/Colab Notebooks/6c_char_clip_time.txt',np_times)
plt.figure(1)
plt.plot(range(epochs), test_acc_pts)
plt.xlabel(str(epochs) + ' iterations')
plt.ylabel('Accuracy against Test Data')
plt.figure(2)
plt.plot(range(epochs), training_cost_pts)
plt.xlabel(str(epochs) + ' iterations')
plt.ylabel('Training Cost')
plt.figure(3)
plt.plot(range(epochs), training_acc_pts)
plt.plot(range(epochs), test_acc_pts)
plt.xlabel(str(epochs) + ' iterations')
plt.ylabel('Accuracy ')
plt.legend(["Training Accuracy","Test Accuracy"])
plt.show()
sess.close()
- [Fashion MNIST with Keras and TPUs](https://research.google.com/seedbank/seed/fashion_mnist_with_keras_and_tpus): Classify fashion-related images with deep learning.
- [DeepDream](https://research.google.com/seedbank/seed/deepdream): Produce DeepDream images from your own photos.
- [Convolutional VAE](https://research.google.com/seedbank/seed/convolutional_vae): Create a generative model of handwritten digits.
"""
import tensorflow as tf
import matplotlib.pyplot as plt
# %matplotlib inline
from tqdm import tqdm
import numpy as np
import os
from random import shuffle
import cv2
from google.colab import drive
drive.mount("/content/drive", force_remount=True)
train_file = "/content/drive/My Drive/Colab Notebooks/train.zip"
test_file = "/content/drive/My Drive/Colab Notebooks/test.zip"
import zipfile
with zipfile.ZipFile(train_file, 'r') as z:
z.extractall()
with zipfile.ZipFile(test_file, 'r') as z:
z.extractall()
ls
TEST_DIR = "./test"
TRAIN_DIR = "./train"
LEARNING_RATE = 1e-3
try:
text = unicode(text, 'utf-8')
except NameError: # unicode is a default on python 3
pass
text = unicodedata.normalize('NFD', text)\
.encode('ascii', 'ignore')\
.decode("utf-8")
return str(text)
## Load google drive content
from google.colab import drive
drive.mount('/content/drive')
## Load Mercari full data
full_data_d=pd.read_csv('/content/drive/My Drive/train_mercari.csv')
# Drop train_id attribute
try:
full_data_d.drop("train_id", axis=1, inplace=True)
except:
print "Already removed train_id"
# Add index to each products
index_product=range(full_data_d.shape[0])
import requests
from bs4 import BeautifulSoup
import time
import json
import csv
import re
import pandas as pd
from multiprocessing import Process
from google.colab import drive
drive.mount('/drive')
def collect_data(ad):
car_url = ad
headers = {
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'Accept-Encoding':
'gzip, deflate, br',
'Accept-Language':
'ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36'
}
cookies = {'autoru_gdpr': '1'}
response = requests.get(car_url, headers=headers, cookies=cookies)
response.encoding = 'utf-8'
page = BeautifulSoup(response.text, 'lxml')
Original file is located at
https://colab.research.google.com/drive/15jIEz03XWJUyGDafKt8LgUaxGQb-OFdY
"""
#Importing the libraries required
from google.colab import drive
!ls "/content/gdrive/My Drive/CIS 519 project"
from google.colab.patches import cv2_imshow
import numpy as np
import cv2
import matplotlib.pyplot as plt
import math
import time
drive.mount('/content/gdrive') #mounting onto Google drive
#Function to resize the images/frames
def imagesresize(images):
print("Resizing all images")
resized_images=[]
for image in images:
dim=(100,100)
resized = cv2.resize(image, dim, interpolation = cv2.INTER_AREA) #resizing all images to given dimensions
resized_images.append(resized)
return(resized_images)
#Function to crop the images/frames
def crop_frame(images):
print("Cropping all images")
cropped_images=[]
def main():
############## Hyperparameters ##############
load_weights = False # If you want to load the agent, set this to True
save_weights = False # If you want to save the agent, set this to True
training_mode = True # If you want to train the agent, set this to True. But set this otherwise if you only want to test it
reward_threshold = 300 # Set threshold for reward. The learning will stop if reward has pass threshold. Set none to sei this off
using_google_drive = False
render = True # If you want to display the image, set this to True. Turn this off if you run this in Google Collab
n_update = 1024 # How many episode before you update the Policy. Recommended set to 128 for Discrete
n_plot_batch = 100000000 # How many episode you want to plot the result
n_episode = 100000 # How many episode you want to run
n_saved = 10 # How many episode to run before saving the weights
policy_kl_range = 0.03 # Set to 0.0008 for Discrete
policy_params = 5 # Set to 20 for Discrete
value_clip = 1.0 # How many value will be clipped. Recommended set to the highest or lowest possible reward
entropy_coef = 0.0 # How much randomness of action you will get
vf_loss_coef = 1.0 # Just set to 1
batchsize = 32 # How many batch per update. size of batch = n_update / batchsize. Rocommended set to 4 for Discrete
PPO_epochs = 10 # How many epoch per update
n_aux_update = 5
max_action = 1.0
gamma = 0.99 # Just set to 0.99
lam = 0.95 # Just set to 0.95
learning_rate = 3e-4 # Just set to 0.95
#############################################
#env_id = '3DBall'
#env = default_registry[env_id].make()
env = UnityEnvironment(file_name=None, seed=1)
env.reset()
behavior_name = list(env.behavior_specs)[0]
behavior_spec = env.behavior_specs[behavior_name]
print(behavior_spec)
state_dim = behavior_spec.observation_shapes[0][0]
action_dim = behavior_spec.action_size
print('state_dim: ', state_dim)
print('action_dim: ', action_dim)
agent = Agent(state_dim, action_dim, training_mode, policy_kl_range,
policy_params, value_clip, entropy_coef, vf_loss_coef,
batchsize, PPO_epochs, gamma, lam, learning_rate)
runner = Runner(env, agent, render, training_mode, n_update, n_aux_update,
max_action)
#############################################
if using_google_drive:
from google.colab import drive
drive.mount('/test')
if load_weights:
agent.load_weights()
print('Weight Loaded')
rewards = []
batch_rewards = []
batch_solved_reward = []
times = []
batch_times = []
for i_episode in range(1, n_episode + 1):
total_reward, time = runner.run_episode()
print('Episode {} \t t_reward: {} \t time: {} \t '.format(
i_episode, total_reward, time))
batch_rewards.append(int(total_reward))
batch_times.append(time)
if save_weights:
if i_episode % n_saved == 0:
agent.save_weights()
print('weights saved')
if reward_threshold:
if len(batch_solved_reward) == 100:
if np.mean(batch_solved_reward) >= reward_threshold:
print('You solved task after {} episode'.format(
len(rewards)))
break
else:
del batch_solved_reward[0]
batch_solved_reward.append(total_reward)
else:
batch_solved_reward.append(total_reward)
if i_episode % n_plot_batch == 0 and i_episode != 0:
# Plot the reward, times for every n_plot_batch
plot(batch_rewards)
plot(batch_times)
for reward in batch_rewards:
rewards.append(reward)
for time in batch_times:
times.append(time)
batch_rewards = []
batch_times = []
print('========== Cummulative ==========')
# Plot the reward, times for every episode
plot(rewards)
plot(times)
print('========== Final ==========')
# Plot the reward, times for every episode
for reward in batch_rewards:
rewards.append(reward)
for time in batch_times:
times.append(time)
plot(rewards)
plot(times)
def main():
############## Hyperparameters ##############
using_google_drive = False # If you using Google Colab and want to save the agent to your GDrive, set this to True
load_weights = False # If you want to load the agent, set this to True
save_weights = False # If you want to save the agent, set this to True
training_mode = True # If you want to train the agent, set this to True. But set this otherwise if you only want to test it
reward_threshold = None # Set threshold for reward. The learning will stop if reward has pass threshold. Set none to sei this off
render = False # If you want to display the image. Turn this off if you run this in Google Collab
n_update = 2048 # How many episode before you update the Policy
n_plot_batch = 100 # How many episode you want to plot the result
n_episode = 10000 # How many episode you want to run
#############################################
env_name = "BipedalWalker-v2"
env = gym.make(env_name)
state_dim = env.observation_space.shape[0]
action_dim = env.action_space.shape[0]
utils = Utils()
agent = Agent(state_dim, action_dim, training_mode)
#############################################
if using_google_drive:
from google.colab import drive
drive.mount('/test')
if load_weights:
agent.load_weights()
print('Weight Loaded')
'''else :
agent.lets_init_weights()
print('Init Weight')'''
if torch.cuda.is_available() :
print('Using GPU')
rewards = []
batch_rewards = []
batch_solved_reward = []
times = []
batch_times = []
t_updates = 0
for i_episode in range(1, n_episode):
total_reward, time, t_updates = run_episode(env, agent, state_dim, render, training_mode, t_updates, n_update)
print('Episode {} \t t_reward: {} \t time: {} \t '.format(i_episode, total_reward, time))
batch_rewards.append(total_reward)
batch_times.append(time)
if training_mode:
if save_weights:
agent.save_weights()
print('Weights saved')
if reward_threshold:
if len(batch_solved_reward) == 100:
if np.mean(batch_solved_reward) >= reward_threshold :
for reward in batch_times:
rewards.append(reward)
for time in batch_rewards:
times.append(time)
print('You solved task after {} episode'.format(len(rewards)))
break
else:
del batch_solved_reward[0]
batch_solved_reward.append(total_reward)
else:
batch_solved_reward.append(total_reward)
if i_episode % n_plot_batch == 0 and i_episode != 0:
# Plot the reward, times for every n_plot_batch
plot(batch_rewards)
plot(batch_times)
for reward in batch_times:
rewards.append(reward)
for time in batch_rewards:
times.append(time)
batch_rewards = []
batch_times = []
print('========== Cummulative ==========')
# Plot the reward, times for every episode
plot(rewards)
plot(times)
print('========== Final ==========')
# Plot the reward, times for every episode
plot(rewards)
plot(times)
from sklearn.preprocessing import MinMaxScaler,StandardScaler
from tqdm import tqdm
from random import shuffle
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch import optim
import cv2
# %matplotlib inline
import warnings
warnings.filterwarnings('ignore')
# importing drive and authentication
from google.colab import drive
drive.mount('/content/drive',force_remount=False)
# Change the location as per your drive
TRAIN_DIR_INFECTED =r'C:\Users\sumasark\Downloads\Kaggle\Maleria\cell_images\cell_images\Parasitized'
TRAIN_DIR_UNINFECTED =r'C:\Users\sumasark\Downloads\Kaggle\Maleria\cell_images\cell_images\Uninfected'
LR_RATE=1e-3
IMG_SIZE=28
MODEL_NAME = 'maleria--{}--{}.model'.format(LR_RATE,'2conv-basic')
# Preprocess raw images into numpy arrays
training_data = []
def create_data_infected():
try:
for img in tqdm(os.listdir(TRAIN_DIR_INFECTED)):
def train_model(model_name, dataset_type, dataset_size, lang, google,
random_seed):
print(f'BEGIN EXPERIMENT')
print(f'model: {model_name}')
print(f'dataset: {dataset_type}')
print(f'size: {dataset_size}')
if google:
drive.mount('/content/drive')
tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
title_fb = FeatureBuilder(['title'])
if lang == 'PL':
dataset_loader = PolishDatasetLoader
elif lang == 'ENG':
dataset_loader = EnglishDatasetLoader
train_df = dataset_loader.load_train(dataset_type, dataset_size)
train_dataset, val_dataset = preprocess_train_val(train_df, title_fb,
tokenizer, random_seed)
test_df = dataset_loader.load_test(dataset_type)
test_dataset = preprocess_test(test_df, title_fb, tokenizer)
logdir_name = f'{model_name}_{dataset_type}_{dataset_size}'
logdir = os.path.join("logs", logdir_name)
train_batch_size = 16
num_train_epochs = 10
half_train = (len(train_dataset) * num_train_epochs) // (2 *
train_batch_size)
if not random_seed:
train_seed = np.random.randint(1_000_000)
else:
train_seed = random_seed
training_args = TrainingArguments(
output_dir='./results',
num_train_epochs=num_train_epochs, # total number of training epochs
per_device_train_batch_size=
train_batch_size, # batch size per device during training
per_device_eval_batch_size=64, # batch size for evaluation
warmup_steps=
half_train, # number of warmup steps for learning rate scheduler
weight_decay=0.01, # strength of weight decay
logging_dir=logdir, # directory for storing logs
logging_steps=10, # for training metrics
disable_tqdm=False, # show some progress
fp16=True, # float 16 acceleration
evaluation_strategy='epoch', # evaluate after epoch
load_best_model_at_end=True, # load best model
metric_for_best_model='eval_f1', # use model with best F1 score
save_total_limit=5, # store last 5 checkpoints
seed=train_seed # random seed
)
model = AutoModelForSequenceClassification.from_pretrained(model_name,
num_labels=2)
trainer = Trainer(
model=model,
args=training_args,
train_dataset=train_dataset,
eval_dataset=val_dataset,
compute_metrics=compute_metrics,
)
print('DEVICE USED: ', training_args.device)
print('TRAINING')
trainer.train()
print('EVALUATION ON VALIDATION SET')
eval = trainer.evaluate()
print(eval)
print('TESTING')
pred = trainer.predict(test_dataset)
metrics = pd.DataFrame(compute_metrics(pred), index=[0])
metrics['model'] = model_name
metrics['dataset_type'] = dataset_type
metrics['dataset_size'] = dataset_size
with pd.option_context('display.max_columns', None):
print(metrics)
print('SAVING MODEL')
model_tmp_save = 'results/test'
model.save_pretrained(model_tmp_save)
if google:
DRIVE = 'drive/MyDrive'
p = Path(
os.path.join(DRIVE, 'MLT4PM', lang, model_name, dataset_type,
dataset_size))
p.mkdir(parents=True, exist_ok=True)
os.system(
f'python -m transformers.convert_graph_to_onnx --model {model_tmp_save} --framework pt --tokenizer {model_name} {p}/model/model.onnx'
)
os.system(f'mv {p}/model/model.onnx {p}/model.onnx')
os.system(f'rm -R {p}/model/')
os.system(f'rm -R {model_tmp_save}')
metrics.to_csv(f'{p}/metrics.csv')
log_path = Path(
os.path.join(DRIVE, "MLT4PM", lang, "logs", logdir_name))
log_path.mkdir(parents=True, exist_ok=True)
os.system(f'cp -R {logdir} {log_path}')
os.system('rm -R ./results')
print('EXPERIMENT ENDED \n\n')
# -*- coding: utf-8 -*-
"""textCNN.ipynb
Automatically generated by Colaboratory.
Original file is located at
https://colab.research.google.com/drive/1bliLih4Evxx1bFW4vaqvCpeDziOkcI1C
"""
from google.colab import drive
drive.mount('/gdrive', force_remount = True)
"""# Import Library"""
import pandas as pd
from pandas import DataFrame as df
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
# nltk
import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords
from nltk.stem import SnowballStemmer
# Word2vec
import gensim
from gensim.models import Word2Vec #@
from gensim.utils import simple_preprocess #@
from gensim.models.keyedvectors import KeyedVectors #@
concatenate2=Concatenate(name="concatenate2")([x3,x1])
outputs=Dense(10,activation="softmax",name="output")(concatenate2)
model3=Model(inputs=inputs,outputs=outputs)
model3.summary()
model3.compile("adamax",loss='categorical_crossentropy',
metrics=['accuracy'])
np.random.seed(9)
model3.fit(flat_input_train,y_train_cat,
validation_data=(flat_input_test,y_test_cat),epochs=15)
import os , sys
from google.colab import drive
drive.mount('/content/drive') #get access to drive
os.chdir("/content/drive/My Drive/PYP") # change working directory
print(os.getcwd())# read current work direktory
#model3.save("Aufgabe4.hdf5")
j=model3.to_json();
with open("A4.json","+w") as f:
f.write(j)
!ls
"""# Dropout !"""
from keras.layers import Dropout
inputs = Input(name="inputs",shape=(flat_input_train.shape[1],))
# -*- coding: utf-8 -*-
"""script_monocular_numpy.ipynb
Automatically generated by Colaboratory.
Original file is located at
https://colab.research.google.com/drive/1OhdR-7rQNOT6tErnWYD1i8CZTd2vdBdU
"""
from google.colab import drive
drive.mount('/data/')
from pathlib import Path
base_dir = ('/data/My Drive')
#!cp /data/My\ Drive/DeepLearning/student_data.zip /content/SimCLR
#!unzip student_data.zip
# Commented out IPython magic to ensure Python compatibility.
#!git clone https://github.com/AmeerHamza111/SimCLR.git
# %cd SimCLR
#!wget https://github.com/Spijkervet/SimCLR/releases/download/1.2/checkpoint_100.tar
#!sh setup.sh || python3 -m pip install -r requirements.txt || exit 1
#!pip install pyyaml --upgrade
#!cp -r /data/My\ Drive/DeepLearning/networks /content/SimCLR
#!cp -r /data/My\ Drive/DeepLearning/models /content/SimCLR
import fnmatch
import os
from google.colab import drive #library to conect to to google driv
from wordcloud import WordCloud #library to visualization
import matplotlib.pyplot as plt #library to visualization
import seaborn as sns #library to visualization
from sklearn.ensemble import RandomForestClassifier # MLA Library
from sklearn.model_selection import train_test_split# MLA Library
from sklearn.model_selection import cross_val_score# MLA Library
from sklearn.metrics import classification_report, confusion_matrix # MLA Library
from sklearn.metrics import roc_curve, auc # MLA Library
from sklearn.preprocessing import LabelEncoder #MLA Library
from keras.models import Sequential #Deep Learning Library
from keras.layers import Dense #Deep Learning Library
from keras.optimizers import SGD,Adam #Deep Learning Library
#------------------------- READ FILES --------------------------
drive.mount('/content/drive') #mount data from drive
!ls "/content/drive/My Drive/umba" #reading the file xls
data = pd.read_excel('/content/drive/My Drive/umba/crunchbase_monthly_export_d43b4klo2ade53.xlsx',sheet_name=None) #xls to data frame
odict_keys=['Companies', 'Rounds', 'Investments', 'Acquisitions'] #names of the sheets
data1=data[odict_keys[0]]
#------------------------- data cleaning ------------------------
category=data1['status'].astype('category') # y variable to describe
data1['founded_at'] = pd.to_datetime(data1['founded_at'], errors = 'coerce' ) #from object to datetime
data_test_model= data1[['market','funding_total_usd','funding_rounds','status']] #select the variables to the model
features=data_test_model.replace(np.nan,-9, regex=True) #avoid nan
features['market'] = features['market'].str.replace('|', ' ') #cleaning data
features['market'] = features['market'].str.replace('+', ' ') #cleaning data
features['market'] = features['market'].str.replace('&', ' ') #cleaning data
return bs.find_all('a')
Alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
df = pd.DataFrame()
for a in Alphabet[0]:
url = "https://www.goldenpages.ie/business/sitemap/" + a
Links = getLinks(url)
# Skip navigation Links
Links = Links[len(Alphabet)+1:]
for link in tqdm(Links):
newCompany = "https://www.goldenpages.ie" + link.attrs['href']
output = output.append(get_company_infos(newCompany), ignore_index=True)
df.head()
# Save CSV to Google Drive
from google.colab import drive
drive.mount('drive')
df.to_csv("goldenpages.csv")
!cp goldenpages.csv drive/My\ Drive/
#!/usr/bin/env python
# coding: utf-8
# <a href="https://colab.research.google.com/github/isanjit3/LungCancer/blob/master/Lung_Cancer_Segmentation_NSCLS_Radiomics.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>
# # Environment Setup
#Specify if you want to enable the test code.
#When you run entire notebook, set it to false.
global_is_test = False
#Mount google Drive.
from google.colab import drive
drive.mount('/gdrive')
#install required packages
get_ipython().system('pip3 install pydicom')
get_ipython().system('pip3 install dicom_contour')
# # Get data files
#
# ## Copy files from Google Drive
#unzip the data fromthe google drive to the VM
#if not global_is_test:
#!unzip '/gdrive/My Drive/Lung/lung-full.zip' -d '/content/lung/'
#Partial data set
#if global_is_test:
#!unzip '/gdrive/My Drive/Lung/lung-partial.zip' -d '/content/lung/'
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(device)
if device == 'cuda':
torch.cuda.empty_cache()
model, optimizer, loss = 0, 0, 0
gc.collect()
#!nvidia-smi
# In[3]:
platform = 'local' #'Kaggle' # 'Colab' # 'gcp'
if platform == 'Colab':
from google.colab import drive
drive.mount("/content/drive", force_remount=False)
# In[4]:
debug = False
#debug = True
effnet_ver = 'b4' #'b4'
dropout_rate = 0.02
is_resume_train = True #False #True
n_samples = 100_000 if not debug else 20
img_h, img_w = 684, 2048
#img_h, img_w = 512, 2048
"""tsp.ipynb
Automatically generated by Colaboratory.
Original file is located at
https://colab.research.google.com/drive/1sQfErpnwjovlPxAF7O0IcASJZFDURe4P
"""
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import sys
from google.colab import drive
# Mount google drive
drive.mount("/content/drive")
"""# Coordinates and euclidean distance"""
class Coordinate:
def __init__(self, x, y):
self.x = x
self.y = y
@staticmethod
def cal_dist(a, b):
return np.sqrt(((a.x - b.x)**2) + ((a.y - b.y)**2))
@staticmethod
def cal_tot_dist(coordinate):
# Initializing distance
import os import time import numpy as np import matplotlib.pyplot as plt import PIL import sys import cv2 as cv import pypdn from IPython.display import clear_output from google.colab import drive mount_folder = '/content/gdrive' drive.mount(mount_folder, True) app_folder = mount_folder + "/My Drive/AnimeNet/" os.chdir(app_folder) TRAIN_PATH = os.path.join(app_folder, 'face_segmented/') TEST_PATH = os.path.join(app_folder, 'face_2/') SAMPLE_DIR = os.path.join(app_folder, 'face_segmented_gen_human') TEST_SIZE = 100 FULL_SIZE = 2000 LAMBDA = 100
# -*- coding: utf-8 -*-
"""insta_follower.ipynb
Automatically generated by Colaboratory.
Original file is located at
https://colab.research.google.com/drive/17mLlHKua6qU9gHnpx7SwOhWHIO_ynzVX
"""
from google.colab import drive # мы импортируем из google colab модуль drive: #import drive module
drive.mount(
'/content/gdrive', force_remount=True
) #мы вызываем функцию drive.mount чтобы подключить колаб к диску #connect the colab to the drive
path777 = "/content/gdrive/My Drive/COLAB_Andemir/homeworks/insta_follower.csv" #записываем путь к файлу #path to the file
with open(path777,
"r") as f: #открываем файл для чтения #open the file for reading
segments = f.readlines() #получаем список строк: #list of strings
dict_inst = {} #создаем новый словарь #create a new dictionary
for i in range(
len(segments[0].split(","))
): #для каждого элемента первого списка строк, который отделен на элементы по запятым #for each element of the first list of strings, which is separated into elements by commas
dict_inst[i] = [
] #создаем для каждого подписчика Elan в качестве значения в словаре пустой список #for each Elan subscriber, write an empty list as a value in the dictionary
for line in segments[
1:]: #для каждого элемента списка строк подписок #for each item in the list of subscription strings
for i in dict_inst: #для каждого аккаунта в словаре #for each account in the dictionary
a = line.split(
","
# -*- coding: utf-8 -*-
"""styletransfer_keras.ipynb
Automatically generated by Colaboratory.
Original file is located at
https://colab.research.google.com/drive/1dcA3QK2pn6lbIxnxmNmSpKh884oWJBBR
"""
from google.colab import drive
drive.mount("./gdrive")
drive_root_dir = "./gdrive/My Drive/"
from keras.preprocessing.image import load_img, save_img,img_to_array
#DONT FORGET CHANGE TO GPU
base_image_path = drive_root_dir + 'Images/n02085620-Chihuahua/n02085620_712.jpg'
style_reference_image_path = drive_root_dir + 'flower17/2/image_0167.jpg'
base_image_path = base_image_path
style_reference_image_path = style_reference_image_path
result_prefix = drive_root_dir + 'style_transfer_result'
iterations = 20
total_variation_weight = 1e-4
style_weight = 1.
content_weight = 0.025
# dimensions of the generated picture.
width, height = load_img(base_image_path).size
img_nrows = 400
# -*- coding: utf-8 -*-
"""2015310908_임성규_hw5.py
Automatically generated by Colaboratory.
Original file is located at
https://colab.research.google.com/drive/1BGNxYv33hBJGLmGZBFf3GAWlnAGa4g10
"""
from google.colab import drive
drive.mount('/content/gdrive/')
import os
import random
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import torchvision.transforms as transforms
import torchvision
import torch.nn.functional as F
from torch.utils.data import DataLoader, SubsetRandomSampler
from torch.nn import CrossEntropyLoss
import cv2
import numpy as np
import matplotlib.pyplot as plt
class Net(nn.Module):
def mount_google_drive():
try:
from google.colab import drive
drive.mount('/content/drive')
except Exception as e:
print("not in googole colab environment!")
# -*- coding: utf-8 -*-
"""ML_Eunimartipynb
Automatically generated by Colaboratory.
Original file is located at
https://colab.research.google.com/drive/13NK17Trlr3OOk5IuBdbxXDCwdSsCVDTc
"""
from google.colab import drive
drive.mount('/gdrive') #Srikoushikkamal
import pandas as pd
csv = pd.read_csv('/gdrive/MyDrive/Data/Test1 - Sheet1.csv')
csv.head()
"""#Duplicates Remoed"""
csv.drop_duplicates(subset="Title", keep=False, inplace=True)
"""#Seperate Code from body"""
New = csv['Body']
"""#Remove stop Words"""
import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords
#stopwords.words('english')
rem = set(stopwords.words('english'))
# -*- coding: utf-8 -*-
"""kmeans_asm.ipynb
Automatically generated by Colaboratory.
Original file is located at
https://colab.research.google.com/drive/1h39UtUj-KshlB2PS5NKNlvT4sWdJgh9x
"""
# mounting drive...
from google.colab import drive
drive.mount("/content/gdrive/")
# retrieving the encodings
import json
# for BoW encodings (10 dimension)
#encodings = json.load(open("/content/gdrive/My Drive/asm_bow_encodings_10.json", "r"))
# for BoW encodings (20 dimension) BEST for BoW
#encodings = json.load(open("/content/gdrive/My Drive/asm_encodings.json", "r"))
# for BoW encodings (100 dimension) NOT so good
#encodings = json.load(open("/content/gdrive/My Drive/asm_bow_encodings_100.json", "r"))
# for BoW encodings with tf (10 dimension) BAD
#encodings = json.load(open("/content/gdrive/My Drive/asm_tf_bow_encodings_10.json", "r"))
# for BoW encodings with term frequency (100 dimension) VERY BAD ONE
#encodings = json.load(open("/content/gdrive/My Drive/asm_encodings_100.json", "r"))
# for tfidf encodings (50 dimension) GOOD
# encodings = json.load(open("/content/gdrive/My Drive/asm_tfidf_encodings.json", "r"))
# for tfidf encodings (100 dimension) EVEN BETTER
# encodings = json.load(open("/content/gdrive/My Drive/asm_tfidf_encodings_100.json", "r"))
def mount_gdrive():
"""Mounts the user's Google Drive in Colaboratory."""
assert 'google.colab' in sys.modules, "You must be in Colaboratory to mount your Google Drive"
drive.mount('/content/drive')
Traffic environment semi-supervised Learning Contest
## Goals
The objective is to train a model using images captured by six different cameras attached to the same car to generate a top down view of the surrounding area. The performance of the model will be evaluated by (1) the ability of detecting objects (like car, trucks, bicycles, etc.) and (2) the ability to draw the road map layout.
## Data
You will be given two sets of data:
1. Unlabeled set: just images
2. Labeled set: images and the labels(bounding box and road map layout)
This notebook will help you understand the dataset.
"""
from google.colab import drive
drive.mount('/data/')
from pathlib import Path
base_dir = ('/data/My Drive')
#!pip install numpy==1.17.4
#import numpy as np
#np.__version__
#!cp /data/My\ Drive/DeepLearning/student_data.zip /content/
#!unzip student_data.zip
import os
import random
import numpy as np
vocab.fit_on_texts(train_tgt)
vocab.fit_on_texts(eval_tgt)
logger.info('Vocab Size : {}\n'.format(len(vocab.word_index)))
train_input = list(zip(train_nodes, train_labels, train_node1, train_node2))
eval_input = list(zip(eval_nodes, eval_labels, eval_node1, eval_node2))
test_input = list(zip(test_nodes, test_labels, test_node1, test_node2))
train_set = list(zip(train_input, train_tgt))
eval_set = list(zip(eval_input, eval_tgt))
logger.info('Train and eval dataset size : {} {} '.format(len(train_set), len(eval_set)))
if args.use_colab is not None:
from google.colab import drive
drive.mount('/content/gdrive', force_remount=True)
OUTPUT_DIR = '/content/gdrive/My Drive/data/processed_graphs/{}/{}'.format(args.lang,
args.model)
if not os.path.isdir(OUTPUT_DIR):
os.makedirs(OUTPUT_DIR)
# save the vocab file
os.makedirs(('vocabs/gat/{}'.format(args.lang)), exist_ok=True)
with open(('vocabs/gat/{}/src_vocab'.format(args.lang)), 'wb+') as fp:
pickle.dump(vocab, fp)
else:
OUTPUT_DIR = 'data/processed_graphs/{}/{}'.format(args.lang,
args.model)
if not os.path.isdir(OUTPUT_DIR):
os.makedirs(OUTPUT_DIR)
# save the vocab file
Politics: 0
Technology: 1
Entertainment: 2
Business: 3
Published @ [www.analyticsindiamaga.com](https://analyticsindiamag.com/step-by-step-guide-to-implementing-multi-class-classification-with-bert-tensorflow/)
##Mounting Google Drive
---
Here I have uploaded the dataset in to my Google Drive folder. To access the datasets we must first mount the drive in google colab. Type in and enter the following code to authenticate and mount your Google drive on to colab.
"""
from google.colab import drive
drive.mount("/GD")
"""## Importing Necessary Libraries"""
import pandas as pd
import tensorflow as tf
import tensorflow_hub as hub
from datetime import datetime
from sklearn.model_selection import train_test_split
import os
print("tensorflow version : ", tf.__version__)
print("tensorflow_hub version : ", hub.__version__)
#Installing BERT module
!pip install bert-tensorflow
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten, Dropout
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras import utils
from tensorflow.keras.layers import Conv2D, MaxPooling2D, BatchNormalization
from sklearn.model_selection import StratifiedKFold
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
# **LIBERA ACESSO AO GOOGLE DRIVE**
# In[3]:
from google.colab import drive
drive.mount('/content/drive/', force_remount=True)
# Diretório de arquivos para treinamento:
# In[3]:
#Para Google Coolab:
#ZIP_PATH = '/content/drive/My Drive/Colab Notebooks/Data_Test/NEU - Steel Superficial Defects/NEU surface defect database.zip'
#IMAGE_PATH = '/content/drive/My Drive/Colab Notebooks/Data_Test/NEU - Steel Superficial Defects/TEMPORARIO/NEU surface defect database'
#Para windowns:
ZIP_PATH = 'C:/Users/JG/Desktop/RNA GUND/Codigo/'
IMAGE_PATH = 'C:/Users/GUND/Desktop/RNA GUND/Codigo/NEU surface defect database/'
# EXTRAI ARQUIVO COMPACTADO PARA PASTA DO GOOGLE DRIVE (É NECESSÁRIO EXTRAIR SOMENTE UMA VEZ!!)
from keras.layers import Conv2D, MaxPooling2D, AveragePooling2D # These layers will be used for extracting features and reducing resolution to extract features more deeply
from keras.layers import Dense, Activation, Dropout, Flatten # These layers will be used to prevent overfitting and in making of fully connected layers
from keras.preprocessing import image # We import image library for processing image
from keras.preprocessing.image import ImageDataGenerator # ImageDataGenerator will be used to generate image from pixel values from 'fer2013.csv' dataset
import matplotlib.pyplot as plt # We will use pyplot for creating graph and labelling it as 'plt'
from matplotlib import pyplot
from matplotlib.patches import Rectangle # We use Rectangle to create window boundary around face detected in image
from matplotlib.patches import Circle # We use Circle to mark keypoints like left eye,right eye,nose and mouth on the face
from mtcnn.mtcnn import MTCNN # Multi Task Cascaded Convolutional Neural network,deep learning model will be used for identifying
# the coordinates of face and keypoints
from google.colab import drive # We import drive from google to use dataset and image
drive.mount('/content/drive') # Mounting drive to use files while inputting
!unzip -uq "/content/drive/My Drive/ZIPPED FILE/fer2013.zip" -d "/content/drive/My Drive/UNZIPPED FILE" #Unzipping the zip file of fer2013.zip into UNZIPPED FILE folder
!unzip -uq "/content/drive/My Drive/ZIPPED FILE/dif1.zip" -d "/content/drive/My Drive/UNZIPPED FILE" #Unzipping the zip file of dif1.zip into UNZIPPED FILE folder
!unzip -uq "/content/drive/My Drive/ZIPPED FILE/different.zip" -d "/content/drive/My Drive/UNZIPPED FILE" #Unzipping the zip file of different.zip into UNZIPPED FILE folder
config=tf.ConfigProto(device_count={'GPU':0,'CPU':56}) # We will use CPU for this project and start a session for graph
SESS=tf.Session(config=config)
keras.backend.set_session(SESS)
num_classes=7 # We have 7 emotions ,so we define classes equal to 7
batch_size=256 # At one at a time we will sample 256 instances in convolutional neural network
epochs=50 # Training will be done for 20 epochs
