def predict(batch: Batch):
# Remove empty texts
batch.texts = list(filter(lambda x: len(x) > 0, batch.texts))
model = ModelWrapper()
res = model(batch)
return {"documents": res}
def prepare_model(learning_rate, momentum, checkpoint_file):
"""Prepare a ResNet-34 model with CrossEntropyLoss and SGD.
Args:
learning_rate (float): The learning rate for SGD.
momentum (float): The momentum for SGD.
checkpoint_file (str or None): If not `None`, the path of the
checkpoint file to load.
Returns:
model.Model: The prepared model object.
"""
# Load model.
resnet = torchvision.models.resnet34()
resnet.conv1 = torch.nn.Conv2d(1,
64,
kernel_size=3,
stride=1,
padding=1,
bias=False)
resnet.avgpool = torch.nn.AvgPool2d(2)
# Prepare loss function and optimizer.
loss_function = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(resnet.parameters(),
lr=learning_rate,
momentum=momentum)
# Wrap model object and load checkpoint file if provided.
model = ModelWrapper(resnet, loss_function, optimizer)
if checkpoint_file:
model.load(checkpoint_file)
return model
def test_can_fit_model(self):
""" This test check ability of fitting model in PER to random vector. """
state_shape = (4, )
action_space = 2
model = PrioritizedExperienceReplayTests._create_model(
state_shape, action_space)
PER = PrioritizedExperienceReplay(maxlen=1,
model=model,
key_scaling=10,
gamma=1)
model_wrapper = ModelWrapper(
model=model, optimizer=K.optimizers.Adam(learning_rate=0.01))
model_wrapper.compile()
sample = Sample(action=np.random.randint(0, action_space),
state=np.random.rand(state_shape[0]),
reward=10,
next_state=None)
PER.add(samples=[sample])
history_of_loss = []
fit_vector = np.zeros((action_space, ))
fit_vector[sample.action] = sample.reward
for _ in range(100):
model_wrapper.fit(sample.state, fit_vector)
history_of_loss.append(PER._loss_calculate(sample=sample))
for idx, loss in enumerate(history_of_loss[:-1]):
self.assertGreater(loss, history_of_loss[idx + 1])
def train(args):
loss_fn = nn.CrossEntropyLoss(ignore_index=0)
adaptive = ModelWrapper(args, loss_fn, get_loader)
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad,
adaptive.model.parameters()),
lr=args.learning_rate,
betas=(args.alpha, args.beta),
weight_decay=args.l2_rate)
if adaptive.train(optimizer, args):
return 0
def main(args):
loss_fn = nn.CrossEntropyLoss(ignore_index=0)
adaptive = ModelWrapper(args, loss_fn)
with open(args.word2idx_path, 'r') as fr:
word2idx = json.loads(fr.read())
with open(args.sememe2idx_path, 'r') as fr:
sememe2idx = json.loads(fr.read())
results = ResDataset(args.gen_file_path, word2idx, sememe2idx)
res_loader = data.DataLoader(dataset=results, batch_size=1, shuffle=False)
scores = adaptive.score(res_loader)
with codecs.open(args.output_path, 'w', 'utf-8') as fw:
fw.write('\n'.join(scores))
return 0
def __init__(self, environment: Env,
memory: AbstractMemory,
policy: AbstractPolicy,
model: K.Model,
optimizer: K.optimizers.Optimizer,
logger: Logger):
self.environment = environment
self.memory = memory
self.policy = policy
self.model = ModelWrapper(model, optimizer)
self.logger = logger
self.history = []
logger.create_settings_model_file(model)
logger.create_settings_agent_file(self)
def get(self, id):
m = ModelWrapper("pickle_model.pkl")
customer = data[data.cuid == id]
if len(customer) == 0:
print("Customer not found")
return {"conv": 0, "revenue": 0}
print("Customer found")
res = m.predict(customer)
print("res[0]", res[0], "res[1]", res[1])
mes = metrics(customer)
return {
"conv": res[0][0],
"revenue": str(res[1].values[0]),
"message": mes
}
def __init__(self):
""" Khởi động ứng dụng """
super(UiOutputDialog, self).__init__()
self.model_wrapper = ModelWrapper(model_path)
self.capture = None
self.timer = QTimer(self)
self.timer.timeout.connect(self.update_frame)
loadUi("window.ui", self)
self.holistic = mp.solutions.holistic.Holistic()
self.face = mp.solutions.face_mesh.FaceMesh(max_num_faces=100)
self.hand = mp.solutions.hands.Hands(max_num_hands=100)
now = QDate.currentDate()
current_date = now.toString('ddd dd MMMM yyyy')
current_time = datetime.datetime.now().strftime("%I:%M %p")
self.Date_Label.setText(current_date)
self.Time_Label.setText(current_time)
self.image = None
self.settings = UISettings()
self.Settings_Button.clicked.connect(
lambda: self.handle_setting_button())
self.warnings = []
self.prev_waring_code = -1
self.prev_waring_code_1 = -1
self.prev_waring_code_2 = -1
self.warning_history = UIWarnings()
self.Warning_History.clicked.connect(
lambda: self.handle_waring_history_button())
self.net = cv2.dnn.readNet("yolov3-tiny.weights", "yolov3-tiny.cfg")
with open("yolov3.txt", 'r') as f:
self.classes = [line.strip() for line in f.readlines()]
self.colors = np.random.uniform(0, 255, size=(len(self.classes), 3))
def __init__(self,
environment: Env,
memory: AbstractMemory,
policy: AbstractPolicy,
model: K.Model,
logger: Logger,
gamma: float,
optimizer: K.optimizers.Optimizer,
n_step: int = 1):
self.model = ModelWrapper(model, optimizer)
#self.model.compile()
self.current_model = None
self.gamma = gamma
self.n_step = n_step
super(DQN, self).__init__(environment=environment,
memory=memory,
policy=policy,
model=model,
optimizer=optimizer,
logger=logger)
def generate(args):
# Load word2idx
adaptive = ModelWrapper(args, data_loader=get_loader)
if adaptive.generate(args):
return 0
shuffle=True,
collate_fn=data.image_label_list_of_masks_collate_function)
validation_dataset_fid = DataLoader(
data.Places365(path_to_index_file=args.path_to_places365,
index_file_name='val.txt',
max_length=6000,
validation=True),
batch_size=args.batch_size,
num_workers=args.batch_size,
shuffle=False,
collate_fn=data.image_label_list_of_masks_collate_function)
validation_dataset = data.Places365(
path_to_index_file=args.path_to_places365, index_file_name='val.txt')
# Init model wrapper
model_wrapper = ModelWrapper(
generator=generator,
discriminator=discriminator,
vgg16=vgg16,
training_dataset=training_dataset,
validation_dataset=validation_dataset,
validation_dataset_fid=validation_dataset_fid,
generator_optimizer=generator_optimizer,
discriminator_optimizer=discriminator_optimizer)
# Perform training
if args.train:
model_wrapper.train(epochs=args.epochs, device=args.device)
# Perform testing
if args.test:
print('FID=', model_wrapper.validate(device=args.device))
model_wrapper.inference(device=args.device)
RNN(HIDDEN_SIZE),
layers.RepeatVector(3),
RNN(128, return_sequences=True),
layers.TimeDistributed(layers.Dense(len(CHARS), activation='softmax'))
])
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.summary()
train_generator = encode_generator(training_generator, BATCH_SIZE)
hist = model.fit_generator(train_generator,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
verbose=1,
use_multiprocessing=True,
workers=-2,
callbacks=callbacks,
validation_data=train_generator, validation_steps=30)
score = model.evaluate_generator(encode_generator(
test_generator, BATCH_SIZE), steps=STEPS_PER_EPOCH)
print(score)
config = build_config(MODEL_NAME, LEARNING_RATE, BATCH_SIZE,
EPOCHS, STEPS_PER_EPOCH, score[0], score[1])
wrapper = ModelWrapper(model, config=config)
wrapper.save_model()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 17/11/14 PM2:21
# @Author : shaoguang.csg
# @File : main.py
from parse_conf import DataConf, ModelConf
from model_wrapper import ModelWrapper
from utils.logger import logger
from time import time
start = time()
data_conf = DataConf()
model_conf = ModelConf()
model = ModelWrapper(data_conf=data_conf, model_conf=model_conf)
model.train()
logger.info(model.get_weight())
model.evaluate()
result = model.predict()
end = time()
logger.info('time: {}'.format(end - start))
# 2 core 1 threads 116
# 1 core 228
def main():
# Grab the dataset from scikit-learn
data = datasets.load_iris()
X = data['data']
y = data['target']
target_names = data['target_names']
feature_names = [
f.replace(' (cm)', '').replace(' ', '_') for f in data.feature_names
]
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.3,
random_state=42)
# Build and train the model
model = RandomForestClassifier(random_state=101)
model.fit(X_train, y_train)
print("Score on the training set is: {:2}".format(
model.score(X_train, y_train)))
print("Score on the test set is: {:.2}".format(model.score(X_test,
y_test)))
# CHANGES HERE >>>>>>>>>>
X_mean = X_train.mean(axis=0).round(1)
feature_defaults = dict(zip(feature_names, X_mean.tolist()))
wrapped = ModelWrapper(model_name='iris-rf',
model_version=MODEL_VERSION,
model_object=model,
class_labels=target_names.tolist(),
feature_defaults=feature_defaults)
# Save the model
model_filename = 'iris-rf-v{}.pkl'.format(MODEL_VERSION)
print("Saving model to {}...".format(model_filename))
joblib.dump(wrapped, model_filename)
# ***** Generate test data *****
print('Generating test data...')
all_probs = model.predict_proba(X_test)
all_test_cases = prep_test_cases(X_test, all_probs, feature_names,
target_names)
test_data_fname = 'testdata_iris_v{}.json'.format(MODEL_VERSION)
with open(test_data_fname, 'w') as fout:
json.dump(all_test_cases, fout)
# ***** Generate test data (Missing) *****
print('Generating test data with missing values...')
missing_grps = [(0, ), (1, ), (2, ), (0, 1), (0, 2), (1, 2), (0, 1, 2)]
X_mean = X_train.mean(axis=0).round(1)
all_features = []
all_probs = []
for missing_cols in missing_grps:
X_missing = X_test.copy().astype('object')
X_scored = X_test.copy()
for col in missing_cols:
X_missing[:, col] = None
X_scored[:, col] = X_mean[col]
# Use the imputed one to find expected probabilities
all_probs.extend(model.predict_proba(X_scored))
all_features.extend(X_missing)
all_test_cases_missing = prep_test_cases(all_features, all_probs,
feature_names, target_names)
test_data_fname = 'testdata_iris_missing_v{}.json'.format(MODEL_VERSION)
with open(test_data_fname, 'w') as fout:
json.dump(all_test_cases_missing, fout)
shuffle=True)
validation_dataset = DataLoader(
CellInstanceSegmentation(path=os.path.join(args.path_to_data, "val"),
augmentation_p=0.0, two_classes=not args.three_classes),
collate_fn=collate_function_cell_instance_segmentation, batch_size=1, num_workers=1, shuffle=False)
test_dataset = DataLoader(
CellInstanceSegmentation(path=os.path.join(args.path_to_data, "test"),
augmentation_p=0.0, two_classes=not args.three_classes),
collate_fn=collate_function_cell_instance_segmentation, batch_size=1, num_workers=1, shuffle=False)
# Model wrapper
model_wrapper = ModelWrapper(detr=detr,
detr_optimizer=detr_optimizer,
detr_segmentation_optimizer=detr_segmentation_optimizer,
training_dataset=training_dataset,
validation_dataset=validation_dataset,
test_dataset=test_dataset,
loss_function=InstanceSegmentationLoss(
segmentation_loss=SegmentationLoss(),
ohem=args.ohem,
ohem_faction=args.ohem_fraction),
device=device)
# Perform training
if args.train:
model_wrapper.train(epochs=args.epochs,
optimize_only_segmentation_head_after_epoch=args.only_train_segmentation_head_after_epoch)
# Perform validation
if args.val:
model_wrapper.validate(number_of_plots=30)
# Perform testing
if args.test:
model_wrapper.test()
if not os.path.exists(processed_data_dir):
os.makedirs(processed_data_dir)
train_set_processed_data_path = os.path.join(processed_data_dir,
'train_{}.npy'.format(img_size))
if os.path.exists(train_set_processed_data_path):
train_set_data = np.load(train_set_processed_data_path)
print('Data loaded!')
else:
train_set_data = process_data.process_train_set_data(
img_size, train_set_raw_data_dir, train_set_processed_data_path)
print('Data processed!')
model_wrapper = ModelWrapper(learning_rate,
img_size,
tensorboard_dir=tensorboard_dir)
model = model_wrapper.model
model_path = os.path.join(model_dir, model_wrapper.name)
if os.path.exists(model_path):
model.load(os.path.join(model_path, model_wrapper.name))
print('Model loaded!')
train_set_data, validation_set_data = train_set_data[:-500], train_set_data[
-500:]
train_x = np.array([i[0]
for i in train_set_data]).reshape(-1, img_size, img_size,
1)
train_y = [i[1] for i in train_set_data]
model_list = os.listdir(model_dir)
print('\nFind model:')
for num, model in enumerate(model_list):
print(num, model)
model_name = model_list[int(input('\nChoose a model (enter a number): '))]
print('Load Model {}'.format(model_name))
model_path = os.path.join(model_dir, model_name)
learning_rate = float(model_name.split('-')[2])
img_size = int(model_name.split('-')[3])
model = ModelWrapper(learning_rate, img_size).model
model.load(os.path.join(model_path, model_name))
print('Model loaded!')
test_set_processed_data_path = os.path.join(processed_data_dir,
'test_{}.npy'.format(img_size))
if os.path.exists(test_set_processed_data_path):
test_set_data = np.load(test_set_processed_data_path)
print('Data loaded!')
else:
test_set_data = process_data.process_test_set_data(
img_size, test_set_raw_data_dir, test_set_processed_data_path)
print('Data processed!')
fig = plt.figure()
# # data dataset = Dataset() dataset.load_vocab_tokens_and_emb() # # config = ModelSettings() config.vocab = dataset.vocab config.model_tag = model_tag config.model_graph = build_graph config.is_train = False config.check_settings() # model = ModelWrapper(config) model.prepare_for_prediction() # text_raw = ["这本书不错"] """ work_book = xlrd.open_workbook(file_raw) data_sheet = work_book.sheets()[0] text_raw = data_sheet.col_values(0) """ # preds_list = [] logits_list = [] # for item in text_raw:
shots = args.shots
token_limit = args.token_limit
model_type = args.model
model_size = args.model_size
data_size = args.data_size
lr_base = args.lr
eval_data_size = args.eval_data_size
eval_batch_size = args.eval_batch_size
train_batch_size = args.train_batch_size
local_rank = args.local_rank
fp16 = args.fp16
device, n_gpu = setup_device(local_rank)
if model_type == "bert":
lm = ModelWrapper('bert',
f'bert-{model_size}-uncased',
token_limit=token_limit,
device=device)
elif model_type == "roberta":
lm = ModelWrapper('roberta',
f'roberta-{model_size}',
token_limit=token_limit,
device=device)
elif model_type == "longformer":
lm = ModelWrapper('longformer',
f'allenai/longformer-{model_size}-4096',
token_limit=token_limit,
device=device)
else:
raise KeyError(f"model type {model_type} not supported")
if do_mlm:
classes = [['yes', 'right'], ['maybe'], ['wrong', 'no']]
