def get_gcs_path_of_kaggle_data(data_name, is_private=False):
if is_private:
# Step 1: Get the credential from the Cloud SDK
from kaggle_secrets import UserSecretsClient
user_secrets = UserSecretsClient()
user_credential = user_secrets.get_gcloud_credential()
# Step 2: Set the credentials
user_secrets.set_tensorflow_credential(user_credential)
# Step 3: Use a familiar call to get the GCS path of the dataset
from kaggle_datasets import KaggleDatasets
GCS_DS_PATH = KaggleDatasets().get_gcs_path(data_name)
return GCS_DS_PATH
TEST_DIR = 'E:/KaggleChallenges/cassava-leaf-disease-classification/test_images'
sub = pd.read_csv(f'{BASE_DIR}sample_submission.csv')
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Device:', tpu.master())
tf.config.experimental_connect_to_cluster(tpu)
tf.tpu.experimental.initialize_tpu_system(tpu)
strategy = tf.distribute.experimental.TPUStrategy(tpu)
except:
strategy = tf.distribute.get_strategy()
print('Number of replicas:', strategy.num_replicas_in_sync)
AUTOTUNE = tf.data.experimental.AUTOTUNE()
GCS_PATH = KaggleDatasets().get_gcs_path('cassava-leaf-disease-classification')
GCS_PATH_AUG = KaggleDatasets().get_gcs_path('cassava-aug')
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
IMAGE_SIZE = [512, 512]
CLASSES = ['0', '1', '2', '3', '4']
EPOCHS = 25
TRAINING_FILENAMES = np.array(
tf.io.gfile.glob(GCS_PATH + '/train_tfrecords/ld_train*.tfrec'))
TEST_FILENAMES = np.array(
tf.io.gfile.glob(GCS_PATH + '/test_tfrecords/ld_test*.tfrec'))
AUG_FILENAME = np.array(tf.io.gfile.glob(GCS_PATH_AUG + '/cassva_aug_*.tfrec'))
def count_data_items(filenames):
n = [
def get_gcs_path(comp_name):
return KaggleDatasets().get_gcs_path(comp_name)
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU', tpu.master())
except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
tf.tpu.experimental.initialize_tpu_system(tpu)
strategy = tf.distribute.experimental.TPUStrategy(tpu)
else:
strategy = tf.distribute.get_strategy()
print('Replicas:', strategy.num_replicas_in_sync)
GCS_DS_PATH = KaggleDatasets().get_gcs_path('flower-classification-with-tpus')
MORE_IMAGES_GCS_DS_PATH = KaggleDatasets().get_gcs_path(
'tf-flower-photo-tfrec')
print(GCS_DS_PATH, '\n', MORE_IMAGES_GCS_DS_PATH)
#!ls -l /kaggle/input/tf-flower-photo-tfrec/*/tfrecords-jpeg-224x224/*.tfrec
#!ls -l /kaggle/input/tf-flower-photo-tfrec/imagenet/tfrecords-jpeg-224x224/*.tfrec
#!ls -l /kaggle/input/tf-flower-photo-tfrec/inaturalist/tfrecords-jpeg-224x224/*.tfrec
#!ls -l /kaggle/input/tf-flower-photo-tfrec/openimage/tfrecords-jpeg-224x224/*.tfrec
#!ls -l /kaggle/input/tf-flower-photo-tfrec/oxford_102/tfrecords-jpeg-224x224/*.tfrec
#!ls -l /kaggle/input/tf-flower-photo-tfrec/tf_flowers/tfrecords-jpeg-224x224/*.tfrec
start_time = datetime.now()
print('Time now is', start_time)
end_training_by_tdelta = timedelta(seconds=8400)
this_run_file_prefix = start_time.strftime('%Y%m%d_%H%M_')
print(this_run_file_prefix)
def test_no_token_fails(self):
env = EnvironmentVarGuard()
env.unset(_KAGGLE_USER_SECRETS_TOKEN_ENV_VAR_NAME)
with env:
with self.assertRaises(CredentialError):
client = KaggleDatasets()
def call_get_gcs_path():
client = KaggleDatasets()
with self.assertRaises(BackendError):
gcs_path = client.get_gcs_path()
strategy = tf.distribute.experimental.TPUStrategy(tpu)
else:
# Default distribution strategy in Tensorflow. Works on CPU and single GPU.
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
# In[10]:
AUTO = tf.data.experimental.AUTOTUNE
# Data access
if not localEnvironment and not IN_COLAB:
GCS_DS_PATH = KaggleDatasets().get_gcs_path("jigsaw-multilingual-toxic-comment-classification")
# Configuration
NEXAMPLESPEREPOCH=240000
EPOCHS = 10
if strategy.num_replicas_in_sync==1:
BATCH_SIZE = 4 * strategy.num_replicas_in_sync
else:
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
# MAX_LEN = 512
MODEL = 'jplu/tf-xlm-roberta-large'
if localEnvironment:
MODEL='jplu/tf-xlm-roberta-base'
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
tf.tpu.experimental.initialize_tpu_system(tpu)
strategy = tf.distribute.experimental.TPUStrategy(tpu)
else:
# Default distribution strategy in Tensorflow. Works on CPU and single GPU.
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
SEQUENCE_LENGTH = 128
# Note that private datasets cannot be copied - you'll have to share any pretrained models
# you want to use with other competitors!
GCS_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')
BERT_GCS_PATH = KaggleDatasets().get_gcs_path('bert-multi')
BERT_GCS_PATH_SAVEDMODEL = BERT_GCS_PATH + "/bert_multi_from_tfhub"
TEST_DATASET_SIZE = 63812
def multilingual_bert_model(max_seq_length=SEQUENCE_LENGTH, trainable_bert=True):
"""Build and return a multilingual BERT model and tokenizer."""
input_word_ids = tf.keras.layers.Input(
shape=(max_seq_length,), dtype=tf.int32, name="input_word_ids")
input_mask = tf.keras.layers.Input(
shape=(max_seq_length,), dtype=tf.int32, name="input_mask")
segment_ids = tf.keras.layers.Input(
shape=(max_seq_length,), dtype=tf.int32, name="all_segment_id")
# Load a SavedModel on TPU from GCS. This model is available online at
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ",
len(tf.config.experimental.list_physical_devices('GPU')))
AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}')
GCS_PATH = [None] * FOLDS
GCS_PATH2 = [None] * FOLDS
for i, k in enumerate(IMG_SIZES):
GCS_PATH[i] = KaggleDatasets().get_gcs_path('melanoma-%ix%i' % (k, k))
GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i' % (k, k))
files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] +
'/train*.tfrec')))
files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')))
ROT_ = 180.0
SHR_ = 2.0
HZOOM_ = 8.0
WZOOM_ = 8.0
HSHIFT_ = 8.0
WSHIFT_ = 8.0
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift,
width_shift):
def call_get_gcs_path():
client = KaggleDatasets()
gcs_path = client.get_gcs_path()
if tpu_resolver is None:
if DEBUG:
BATCH_SIZE = 32 * 2
else:
BATCH_SIZE = 32 * 32
elif strategy is not None:
BATCH_SIZE = 32 * strategy.num_replicas_in_sync
if tpu_resolver is None:
DATA_PATH = "/kaggle/input/jigsaw-multilingual-toxic-comment-classification/"
BERT_BASE_DIR = "/kaggle/input/bert-pretrained-models" + \
'/multi_cased_L-12_H-768_A-12' + '/multi_cased_L-12_H-768_A-12'
else:
from kaggle_datasets import KaggleDatasets
GCS_DS_PATH = KaggleDatasets().get_gcs_path(
'jigsaw-multilingual-toxic-comment-classification')
GCS_BERT_PRETRAINED = KaggleDatasets().get_gcs_path('bert-pretrained-models') + \
'/multi_cased_L-12_H-768_A-12'+'/multi_cased_L-12_H-768_A-12'
DATA_PATH = GCS_DS_PATH + '/'
BERT_BASE_DIR = GCS_BERT_PRETRAINED
def pickle_data(max_seq_length=128,
bert_base_dir=BERT_BASE_DIR,
output="features.pkl"):
# --vocab_file="$BERT_BASE_DIR/vocab.txt" \
# --init_checkpoint="$BERT_BASE_DIR/bert_model.ckpt" \
# --bert_config_file="$BERT_BASE_DIR/bert_config.json" \
load_data("pickle",
"/tmp/input.txt",
import pandas as pd
from sklearn.model_selection import KFold
from sklearn.metrics import roc_auc_score
import tensorflow as tf
import tensorflow.keras.backend as K
import efficientnet.tfkeras as efn
import PIL
from kaggle_datasets import KaggleDatasets
from tqdm import tqdm
BASEPATH = "../input/siim-isic-melanoma-classification"
df_train = pd.read_csv(os.path.join(BASEPATH, 'train.csv'))
df_test = pd.read_csv(os.path.join(BASEPATH, 'test.csv'))
df_sub = pd.read_csv(os.path.join(BASEPATH, 'sample_submission.csv'))
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-384x384')
files_train = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')
files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')))
GCS_PATH2 = KaggleDatasets().get_gcs_path('isic2019-384x384')
files_train += tf.io.gfile.glob([GCS_PATH2 + '/train%.2i*.tfrec'%(x*2+1) for x in range(15)]) # 2019
files_train += tf.io.gfile.glob([GCS_PATH2 + '/train%.2i*.tfrec'%(x*2) for x in range(15)]) # 2018"
files_train = np.sort(np.array(files_train))
DEVICE = "TPU"
bs = 16
CFG = dict(
net_count = 7,
batch_size = bs,
slices = paths if labels is None else (paths, labels)
dset = tf.data.Dataset.from_tensor_slices(slices)
dset = dset.map(decode_fn, num_parallel_calls=AUTO)
dset = dset.cache(cache_dir) if cache else dset
dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset
dset = dset.repeat() if repeat else dset
dset = dset.shuffle(shuffle) if shuffle else dset
dset = dset.batch(bsize).prefetch(AUTO)
return dset
strategy = auto_select_accelerator()
BATCH_SIZE = strategy.num_replicas_in_sync * 16 #############WAS 16
GCS_DS_PATH = KaggleDatasets().get_gcs_path("hpa-768768")
GCS_DS_PATH_EXT_DATA = KaggleDatasets().get_gcs_path(
"hpa-public-768-excl-0-16")
load_dir = f"/kaggle/input/hpa-768768/"
df = pd.read_csv(os.getcwd() + 'train_data/df_green.csv')
df["ID"] = df["ID"].str.replace('_green', '')
label_cols = df.columns[2:21]
paths = GCS_DS_PATH + '/' + df['ID']
labels = df[label_cols].values
df_ext = pd.read_csv('train_data/hpa_public_excl_0_16_768.csv', index_col=0)
df_ext = df_ext.drop(['Cellline'], axis=1)
df_ext["Labels_list"] = df_ext["Label"].str.split("|").apply(
lambda x: [int(i) for i in x])
rot=180.0,
shr=2.0,
hzoom=8.0,
wzoom=8.0,
hshift=8.0,
wshift=8.0,
optimizer='adam',
label_smooth_fac=0.05,
tta_steps=25)
BASEPATH = "../input/siim-isic-melanoma-classification"
df_train = pd.read_csv(os.path.join(BASEPATH, 'train.csv'))
df_test = pd.read_csv(os.path.join(BASEPATH, 'test.csv'))
df_sub = pd.read_csv(os.path.join(BASEPATH, 'sample_submission.csv'))
GCS_PATH = KaggleDatasets().get_gcs_path(
'melanoma-%ix%i' % (args["image_size"], args["image_size"]))
files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + "/train*.tfrec")))
files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + "/test*.tfrec")))
if DEVICE == "TPU":
print("connecting to TPU...")
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print("Running on TPU", tpu.master())
except ValueError:
print("Could not connect to TPU")
tpu = None
if tpu:
tf.tpu.experimental.initialize_tpu_system(tpu)
strategy = tf.distribute.experimental.TPUStrategy(tpu)
else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
# %% [markdown]
# # Competition data access
#
# In order to train a model on a TPU, we need to store the dataset in Google Cloud Storage (GCS).
#
# The code cell below will copy the flowers dataset to a GCS bucket co-located with the TPU.
# %% [code]
GCS_DS_PATH = KaggleDatasets().get_gcs_path('tpu-getting-started')
# %% [markdown]
# # Configuration
# %% [code]
IMAGE_SIZE = [512, 512]
EPOCHS = 12
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
GCS_PATH = GCS_DS_PATH + '/tfrecords-jpeg-512x512'
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')
VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')
# %% [code]
#针对不同硬件产生不同配置
AUTO = tf.data.experimental.AUTOTUNE
# Detect hardware, return appropriate distribution strategy
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver() # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.
print('Running on TPU ', tpu.master())
except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
tf.tpu.experimental.initialize_tpu_system(tpu)
strategy = tf.distribute.experimental.TPUStrategy(tpu)
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
print(GCS_DS_PATH)
else:
strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.
print("REPLICAS: ", strategy.num_replicas_in_sync)
# %% [code]
#path、label都作为全局变量
#超参数，根据数据和策略调参
BATCH_SIZE = 1 * strategy.num_replicas_in_sync
img_size = 768
EPOCHS = 25
lr_if_without_scheduler = 0.0003
nb_classes = 3
print('BATCH_SIZE是：',BATCH_SIZE)
def call_get_gcs_path():
client = KaggleDatasets()
gcs_path = client.get_gcs_path()
self.assertEqual(gcs_path, _AUTOML_GCS_BUCKET)
strategy = tf.distribute.get_strategy()
AUTO = tf.data.experimental.AUTOTUNE
BATCH_SIZE = 16 * strategy.num_replicas_in_sync # 根据cpu/tpu自动调整batch大小
IMAGE_SIZE = [512,512]
WIDTH = IMAGE_SIZE[0]
HEIGHT = IMAGE_SIZE[1]
CHANNELS = 3
# 加载官方数据集
try: #Running in Kaggle kernel
from kaggle_datasets import KaggleDatasets
BASE = KaggleDatasets().get_gcs_path('flower-classification-with-tpus')
except ModuleNotFoundError: # Running at my mac
BASE = "/Users/astzls/Downloads/flower"
PATH_SELECT = { # 根据图像大小选择路径
192: BASE + '/tfrecords-jpeg-192x192',
224: BASE + '/tfrecords-jpeg-224x224',
331: BASE + '/tfrecords-jpeg-331x331',
512: BASE + '/tfrecords-jpeg-512x512'
}
IMAGE_PATH = PATH_SELECT[IMAGE_SIZE[0]]
VALIDATION_FILENAMES = tf.io.gfile.glob(IMAGE_PATH + '/val/*.tfrec')
# 统计image数量
AUTO = tf.data.experimental.AUTOTUNE
#超参数定义
#strategy来自tpu_settings.py
BATCH_SIZE = 8 * strategy.num_replicas_in_sync # 根据cpu/tpu自动调整batch大小
LEARNING_RATE = 9.888029308058321e-05
EPOCHS = 8 # 训练周次
IMAGE_SIZE = [512, 512] #手动修改此处图像大小，进行训练
WIDTH = IMAGE_SIZE[0]
HEIGHT = IMAGE_SIZE[1]
CHANNELS = 3
# 加载官方数据集
try: #Running in Kaggle kernel
from kaggle_datasets import KaggleDatasets
BASE = KaggleDatasets().get_gcs_path('flower-classification-with-tpus')
except ModuleNotFoundError: # 本地训练
BASE = "替换成自己的路径"
IMAGE_PATH = BASE + '/tfrecords-jpeg-512x512'
#此处利用tf.io的库函数
#读出文件集方式很多种，也可以用os+re库进行
TRAINING_FILENAMES = tf.io.gfile.glob(IMAGE_PATH + '/train/*.tfrec')
VALIDATION_FILENAMES = tf.io.gfile.glob(IMAGE_PATH + '/val/*.tfrec')
TEST_FILENAMES = tf.io.gfile.glob(IMAGE_PATH + '/test/*.tfrec')
# 牛津大学花蕊数据（原：7310 数据清洗后：2718）Oxford 102 for TPU competition
# 使用后，精确度提升2%
try: #Running in Kaggle kernel
from kaggle_datasets import KaggleDatasets
