def test_save_model(model_name: str, fake_model: Model):
fake_model.__asf_model_name = f"{model_name}:some_old_tag"
save_model(fake_model, "test_save_model")
model = load_model(f"{model_name}:test_save_model")
verify_model_layer_equality(model, fake_model)
def main(folder: str) -> None:
full_path = os.path.join("datasets", folder)
model = load_model("AI_FCN_512")
remove_64(full_path)
run_gui(folder, model)
print("Finishing up...")
break_up_images(full_path)
def train_wrapper(args: Namespace) -> None:
""" Function for training a network. """
model_name = args.model
if args.cont:
model = load_model(model_name)
history = model.__asf_model_history
weights = model.get_weights()
lr_schedule = ExponentialDecay(9.2e-4,
decay_steps=2000,
decay_rate=0.96,
staircase=True)
# optimizer = model.optimizer
model.compile(loss=jaccard_distance_loss,
optimizer=Adam(learning_rate=lr_schedule),
metrics=['accuracy', MeanIoU(num_classes=2)])
model.set_weights(weights)
# model.compile(
# loss='binary_crossentropy', optimizer='adam', metrics=["accuracy"]
# )
else:
model_path = path_from_model_name(model_name)
if not args.overwrite and os.path.isfile(model_path):
print(f"File {model_name} already exists!")
return
# model = create_model_masked(model_name)
model = create_cdl_model_masked(model_name)
history = {'loss': [], 'accuracy': [], "mean_io_u": []}
train_model(model, history, args.dataset, args.epochs)
def predict(config_filename, test_data, vocabulary, char_set, targets,
additionals, rev_categories):
config = Config()
config.load(config_filename)
use_cuda = torch.cuda.is_available()
model, _ = load_model(config.model_filename, config_filename, use_cuda)
model.eval()
competition = config.competition
task_type = config.task_type
task_key = competition + "-" + task_type
test_batches = get_batches(test_data.reviews, vocabulary, char_set, 1,
config.model_config.word_max_count,
config.model_config.char_max_word_length,
targets[task_key], additionals[task_key],
config.model_config.use_pos)
new_reviews = []
for review, batch in zip(test_data.reviews, test_batches):
predictions = model.predict(batch)
if config.model_config.is_sequence_predictor:
length = sum(
[int(elem != 0) for elem in batch.word_indices[0].data])
if model.config.use_crf:
review_pred = predictions[0][:length]
else:
review_pred = predictions[0, :length].cpu()
else:
review_pred = predictions[0].cpu().item()
new_review = get_new_review(review, review_pred, competition,
task_type, rev_categories,
config.model_config.is_sequence_predictor)
new_reviews.append(new_review)
if competition == "imdb":
csv = "id,sentiment\n"
for review in new_reviews:
csv += str(review.rid) + "," + str(review.sentiment) + "\n"
with open(config.output_filename, "w", encoding='utf-8') as f:
f.write(csv)
if competition == "sst2" or competition == "sst1":
text = ""
for review in new_reviews:
text += str(review.sentiment) + " " + review.text + "\n"
with open(config.output_filename, "w", encoding='utf-8') as f:
f.write(text)
if competition == "semeval" or competition == "sentirueval":
xml = '<?xml version="1.0" ?>\n'
xml += '<Reviews>\n' if competition == "semeval" else "<reviews>\n"
for review in new_reviews:
xml += review.to_xml()
xml += '</Reviews>\n' if competition == "semeval" else "</reviews>\n"
with open(config.output_filename, "w", encoding='utf-8') as f:
f.write(xml)
def test_save_model_with_history(model_name: str, fake_model: Model,
new_history: History):
fake_model.__asf_model_name = f"{model_name}:some_old_tag"
save_model(fake_model, "test_save_model", history=new_history)
model = load_model(f"{model_name}:test_save_model")
verify_model_layer_equality(model, fake_model)
assert model.__asf_model_history == new_history
def __prepare(self):
num_classes = len(self.classes)
self.tf_model, self.input_details, self.output_details = load_model(
self.model_path, self.batch_size, self.image_size, num_classes)
self.list_nude_images_paths = get_images_from_folder(
self.nudes_image_folder, self.image_extention)
self.list_non_nude_images_paths = get_images_from_folder(
self.non_nudes_image_folder, self.image_extention)
self.df = pd.DataFrame(columns=['real_label'] + self.classes)
def test_wrapper(args: Namespace) -> None:
model_name = args.model
model = load_model(model_name)
if args.edit:
predictions, data_iter, metadata = test_model_masked(
model, args.dataset, args.edit)
edit_predictions(predictions, data_iter, metadata)
else:
predictions, test_iter = test_model_masked(model, args.dataset,
args.edit)
plot_predictions(predictions, test_iter)
def create_water_mask(model_path: str,
vv_path: str,
vh_path: str,
outfile: str,
verbose: int = 0):
if not os.path.isfile(vv_path):
raise FileNotFoundError(f"Tiff '{vv_path}' does not exist")
if not os.path.isfile(vh_path):
raise FileNotFoundError(f"Tiff '{vh_path}' does not exist")
def get_tiles(img_path):
f = gdal.Open(img_path)
img_array = f.ReadAsArray()
original_shape = img_array.shape
n_rows, n_cols = get_tile_dimensions(*original_shape, tile_size=dems)
padded_img_array = pad_image(img_array, dems)
invalid_pixels = np.nonzero(padded_img_array == 0.0)
img_tiles = stride_tile_image(padded_img_array)
return img_tiles, n_rows, n_cols, invalid_pixels, f.GetProjection(
), f.GetGeoTransform()
# Get vv tiles
vv_tiles, vv_rows, vv_cols, vv_pixels, vv_projection, vv_transform = get_tiles(
vv_path)
# Get vh tiles
vh_tiles, vh_rows, vh_cols, vh_pixels, vh_projection, vh_transform = get_tiles(
vh_path)
model = load_model(model_path)
# Predict masks
masks = model.predict(np.stack((vv_tiles, vh_tiles), axis=3),
batch_size=1,
verbose=verbose)
masks.round(decimals=0, out=masks)
# Stitch masks together
mask = masks.reshape((vv_rows, vv_cols, dems, dems)) \
.swapaxes(1, 2) \
.reshape(vv_rows * dems, vv_cols * dems) # yapf: disable
mask[vv_pixels] = 0
write_mask_to_file(mask, outfile, vv_projection, vv_transform)
# Needed?
f = None
def train_wrapper(args: Namespace) -> None:
""" Function for training a network. """
model_name = args.model
if args.cont:
model = load_model(model_name)
history = model.__asf_model_history
else:
model_path = path_from_model_name(model_name)
if not args.overwrite and os.path.isfile(model_path):
print(f"File {model_name} already exists!")
return
model = create_model_masked(model_name)
history = {"loss": [], "acc": [], "val_loss": [], "val_acc": []}
train_model(model, history, args.dataset, args.epochs)
def main(model_path: str,
vv_path: str,
vh_path: str,
outfile: str,
verbose: int = 0):
if not os.path.isfile(vv_path):
raise FileNotFoundError(f"Tiff '{vv_path}' does not exist")
if not os.path.isfile(vh_path):
raise FileNotFoundError(f"Tiff '{vh_path}' does not exist")
# Get vv tiles
f = gdal.Open(vv_path)
vv_array = f.ReadAsArray()
original_shape = vv_array.shape
n_rows, n_cols = get_tile_dimensions(*original_shape, tile_size=dems)
vv_array = pad_image(vv_array, dems)
invalid_pixels = np.nonzero(vv_array == 0.0)
vv_tiles = tile_image(vv_array)
# Get vh tiles
f = gdal.Open(vh_path)
vh_array = pad_image(f.ReadAsArray(), dems)
vh_tiles = tile_image(vh_array)
model = load_model(model_path)
# Predict masks
masks = model.predict(np.stack((vh_tiles, vv_tiles), axis=3),
batch_size=1,
verbose=verbose)
masks.round(decimals=0, out=masks)
# Stitch masks together
mask = masks.reshape((n_rows, n_cols, dems, dems)) \
.swapaxes(1, 2) \
.reshape(n_rows * dems, n_cols * dems) # yapf: disable
mask[invalid_pixels] = 0
write_mask_to_file(mask, outfile, f.GetProjection(), f.GetGeoTransform())
f = None
def view_filters(model_name):
model = load_model(model_name)
for layer in model.layers:
if not isinstance(layer, Conv2D):
continue
weights, biases = layer.get_weights()
filter_width, filter_height, channels, num_filters = weights.shape
nrows = math.ceil(math.sqrt(num_filters))
ncols = math.ceil(num_filters / nrows)
plt.figure(layer.name)
plt.suptitle(f"Network layer: {layer.name}")
x1w = weights[:, :, 0, :]
for i in range(num_filters):
plt.subplot(nrows, ncols, i + 1)
plt.imshow(x1w[:, :, i], interpolation="nearest", cmap="gray")
plt.show()
def test_wrapper(args: Namespace) -> None:
model_name = args.model
model = load_model(model_name)
if model_type(model) != dataset_type(args.dataset):
print("ERROR: This dataset is not compatible with your model")
return
if dataset_type(args.dataset) == ModelType.MASKED:
predictions, test_iter = test_model_masked(model, args.dataset)
plot_masked_predictions(predictions, test_iter, args.dataset)
else:
details, confusion_matrix = test_model_binary(model, args.dataset)
model_dir = os.path.dirname(path_from_model_name(model_name))
with open(os.path.join(model_dir, 'results.csv'), 'w') as f:
write_dict_to_csv(details, f)
plot_confusion_chart(confusion_matrix)
plot_predictions(details['Percent'], args.dataset)
def train_wrapper(args: Namespace) -> None:
"""Function for training a network"""
data_type = dataset_type(args.dataset)
model_name = args.model
if args.cont:
model = load_model(model_name)
history = model.__asf_model_history
else:
model_path = path_from_model_name(model_name)
if not args.overwrite and os.path.isfile(model_path):
print(f"File {model_name} already exists!")
return
model = create_model(model_name, data_type)
history = {"loss": [], "acc": [], "val_loss": [], "val_acc": []}
if model_type(model) != data_type:
print("ERROR: This dataset is not compatible with your model")
return
train_model(model, history, args.dataset, args.epochs)
def mkdata_wrapper(args: Namespace) -> None:
etl_wm()
setup_data(args.size)
dataset_fpath = f"syntheticTriainingData{date.isoformat(date.today())}"
dataset_dir = os.path.join('datasets', args.directory)
model_name = args.model
model = load_model(model_name)
if args.environment:
final_dataset_fpath = os.path.join(
dataset_dir, f'{args.dataset}_{args.environment}')
dataset = os.path.join(args.directory,
f'{args.dataset}_{args.environment}')
else:
final_dataset_fpath = os.path.join('datasets', args.dataset)
dataset = args.dataset
if not os.path.isdir(dataset_dir):
os.mkdir(dataset_dir)
if not os.path.isdir(final_dataset_fpath):
os.mkdir(final_dataset_fpath)
for folder in os.listdir(dataset_fpath):
for img in os.listdir(os.path.join(dataset_fpath, folder)):
os.rename(os.path.join(dataset_fpath, folder, img),
os.path.join(final_dataset_fpath, img))
shutil.rmtree(dataset_fpath)
move_imgs(final_dataset_fpath)
prepare_data(final_dataset_fpath, 0.2)
predictions, data_iter, metadata = test_model_masked(model,
dataset,
edit=True)
edit_predictions(predictions, data_iter, metadata)
def __init__(self, model_path, image_paths, mode='eval'):
self.model = load_model(model_path, mode)
self.image_paths = image_paths
self.feature = []
self.r = randint(0, len(image_paths))
import time
import cv2
import torch
from PIL import Image
from torch.nn import Softmax
from src.camera import get_camera_capture
from src.display import setup_window, add_label, is_quit_key, is_fullscreen_key, switch_fullscreen_mode
from src.gestures import Gestures
from src.image_processing import get_transform
from src.model import load_model, init_buffer
from src.prediction_smoothing import PredictionSmoothing
if __name__ == "__main__":
model = load_model()
model.eval()
cap = get_camera_capture(0, 320, 240)
full_screen = False
window_name = 'Video Gesture Recognition'
setup_window(window_name)
transform = get_transform()
shift_buffer = init_buffer()
gestures = Gestures()
prediction_smoothing = PredictionSmoothing(7)
softmax = Softmax(1)
while True:
def server_info(token):
servers_names, servers_uuid = sa.get_server_list(token)
# TODO : UI ask specific server name
print("PLEASE SELECT SERVER NAME")
print(servers_names)
index = int(input())
server = sa.get_server_info(token, servers_uuid[index])
print(server['server'])
return servers_uuid[index]
#return data
def get_resource_info(token, uuid):
print("get_resource_info")
res = sa.get_resource_list(token, uuid)
print(res)
return sa.get_mesuare_list(token, res)
if __name__ == '__main__':
token = login('admin', 'devstack')
instance_uuid = server_info(token)
cpu, memory, disk = get_resource_info(token, instance_uuid)
print(cpu, memory, disk)
model = sm.load_model('model/model')
print("PLEASE INPUT RATING DATA")
rating = int(input())
sm.predict(cpu, memory, disk, rating, model)
cors = CORS(
app,
resources={
r"/*": {"origin": "*"},
r"/stackUpdate/*": {"origin": "*"},
r"/login/*": {"origin": "*"},
r"/instanceInfo/*": {"origin": "*"},
r"/setAlarm/*": {"origin": "*"}
}
)
sess = tf.Session()
graph = tf.get_default_graph()
set_session(sess)
model = om.load_model('model/model')
set_session(sess)
@app.route('/', methods=['POST'])
def hello_world():
return 'Hello world'
# 로그인
@app.route("/login", methods=['POST'])
def login():
"""Login Form"""
print("/login <- ")
print(request.get_json())
data = request.get_json()
id = data['id']
password = data['password']
def main():
torch.manual_seed(317)
torch.backends.cudnn.benckmark = True
train_logger = Logger(opt, "train")
val_logger = Logger(opt, "val")
start_epoch = 0
print('Creating model...')
model = get_model(opt.arch, opt.heads).to(opt.device)
optimizer = torch.optim.Adam(model.parameters(), opt.lr)
criterion = CtdetLoss(opt)
print('Loading model...')
if opt.load_model != '':
model, optimizer, start_epoch = load_model(
model, opt.load_model, optimizer, opt.lr, opt.lr_step)
model = torch.nn.DataParallel(model)
# amp
scaler = GradScaler()
print('Setting up data...')
train_dataset = Dataset(opt, 'train')
val_dataset = Dataset(opt, 'val')
train_loader = torch.utils.data.DataLoader(
train_dataset,
batch_size=opt.batch_size,
shuffle=True,
num_workers=16,
pin_memory=True,
drop_last=True
)
val_loader = torch.utils.data.DataLoader(
val_dataset,
batch_size=1,
shuffle=False,
num_workers=1,
pin_memory=True
)
# cal left time
time_stats = TimeMeter(opt.num_epochs, len(train_loader))
for epoch in range(start_epoch + 1, opt.num_epochs + 1):
print('train...')
train(model, train_loader, criterion, optimizer,
train_logger, opt, epoch, scaler, time_stats)
if epoch % opt.val_intervals == 0:
print('val...')
val(model, val_loader, criterion, val_logger, opt, epoch)
save_model(os.path.join(opt.save_dir, f'model_{epoch}.pth'),
epoch, model, optimizer)
# update learning rate
if epoch in opt.lr_step:
lr = opt.lr * (0.1 ** (opt.lr_step.index(epoch) + 1))
print('Drop LR to', lr)
for param_group in optimizer.param_groups:
param_group['lr'] = lr
# without optimizer
save_model(os.path.join(opt.save_dir, 'model_final.pth'), epoch, model)
from data_representation import DeepMatcherProcessor, QqpProcessor
from logging_customized import setup_logging
from src.data_loader import load_data, DataType
from src.model import load_model
from src.prediction import predict
from torch_initializer import initialize_gpu_seed
setup_logging()
if __name__ == "__main__":
args = read_arguments_prediction()
device, n_gpu = initialize_gpu_seed(args.seed)
model, tokenizer = load_model(
os.path.join(args.model_output_dir, args.trained_model_for_prediction),
args.do_lower_case)
model.to(device)
if tokenizer:
logging.info("Loaded pretrained model and tokenizer from {}".format(
args.trained_model_for_prediction))
else:
tokenizer = BertTokenizer.from_pretrained(
args.model_name_or_path, do_lower_case=args.do_lower_case)
logging.info(
"Loaded pretrained model from {} but no fine-tuned tokenizer found, therefore use the standard tokenizer."
.format(args.trained_model_for_prediction))
if args.data_processor == "QqpProcessor":
processor = QqpProcessor()
total_epoch=WARMUP_EPOCHS,
after_scheduler=Da_scheduler_cos)
Db_optimizer = optim.Adam(Db.parameters(), lr=D_MAX_LR, betas=(0.5, 0.999))
Db_scheduler_cos = CosineAnnealingWarmRestarts(Db_optimizer,
T_0=TOTAL_EPOCHS,
T_mult=1,
eta_min=D_MIN_LR)
Db_scheduler = GradualWarmupScheduler(Db_optimizer,
multiplier=1,
total_epoch=WARMUP_EPOCHS,
after_scheduler=Db_scheduler_cos)
if LOAD_MODEL_EPOCH:
G, GF_optimizer, GF_scheduler = load_model(G, GF_optimizer, GF_scheduler,
MODEL_G_DIR, LOAD_MODEL_EPOCH,
DEVICE)
F, GF_optimizer, GF_scheduler = load_model(F, GF_optimizer, GF_scheduler,
MODEL_F_DIR, LOAD_MODEL_EPOCH,
DEVICE)
Da, Da_optimizer, Da_scheduler = load_model(Da, Da_optimizer, Da_scheduler,
MODEL_DA_DIR, LOAD_MODEL_EPOCH,
DEVICE)
Db, Db_optimizer, Db_scheduler = load_model(Db, Db_optimizer, Db_scheduler,
MODEL_DB_DIR, LOAD_MODEL_EPOCH,
DEVICE)
G.to(DEVICE)
F.to(DEVICE)
Da.to(DEVICE)
Db.to(DEVICE)
def test_load_model(model_name: str, fake_model: Model):
with mock.patch('src.model.kload_model', return_value=fake_model):
model = load_model(model_name)
assert model.__asf_model_name == model_name
assert model.__asf_model_history
from typing import Dict, Union
from fastapi import FastAPI
from pydantic import BaseModel
from src.model import load_model, model_predict
model, tokenizer = load_model()
app = FastAPI()
class Ticket(BaseModel):
message: str
@app.post("/ticket_support_classification")
def classify_ticket(ticket: Ticket) -> Dict[str, Union[str, int]]:
assert isinstance(ticket.message, str)
return {
"ticket_message": ticket.message,
"ticket_category": int(model_predict(model, tokenizer, [ticket.message])),
}
def test_wrapper(args: Namespace) -> None:
model_name = args.model
model = load_model(model_name)
# if args.edit:
# predictions, data_iter, metadata = test_model_masked(
# model, args.dataset, args.edit
# )
# edit_predictions(
# predictions, data_iter, metadata
# )
# else:
predictions, test_batch_metadata, input_metadata = test_model_timeseries(
model, args.dataset, args.edit)
model_batch_size = len(test_batch_metadata[0])
current_date_time = str(datetime.utcnow())
# Version number _ Country _ Region
model_name_metadata = model_name.split("_")
metadata = {
"model_test_info": {
"name": model_name,
"model_architecture_version": model_name_metadata[0],
"dataset": args.dataset,
"batch_size": model_batch_size,
"UTC_date_time": current_date_time
}
}
prediction_directory_name = "{0}_{1}_{2}".format(model_name, args.dataset,
current_date_time)
os.mkdir("predictions/{0}".format(prediction_directory_name))
# {
# model_info: {}
# batch_x [
# [
# {
# sample_x: {
# timesteps: [
# [
# test/WA_2018/S1A_ulx_0000_uly_0000.tif
# ]
# ]
# }
# }
# ]
# ]
# }
print(f"length of metadata: {len(test_batch_metadata)}")
print(f"length of predictions: {len(predictions)}")
# fixed = False
# for batches
non_blank_predictions = 0
for idx, batch in enumerate(test_batch_metadata):
metadata["batch_{0}".format(idx)] = []
samples = []
# for sample in batch
for idy, sample in enumerate(batch):
current_prediction_idx = idx * model_batch_size + idy
current_subdataset = sample[0][0].split("/")[
1] #IE: WA_2018, AK_2020
prediction_subdataset_name = f"predictions/{prediction_directory_name}/{current_subdataset}"
if not os.path.isdir(prediction_subdataset_name):
os.mkdir(prediction_subdataset_name)
# if len(predictions) > current_prediction_idx:
image = predictions[current_prediction_idx]
image = np.array(image[:, :, 0].reshape(NETWORK_DEMS, NETWORK_DEMS,
1)).astype(dtype=np.uint8)
# if np.ptp(image) != 0:
# img_0 = array_to_img(image)
prediction_frame = "_".join(sample[0][0].split("_")[-4:])
filename = f"{prediction_subdataset_name}/CDL_{current_subdataset}_prediction_{prediction_frame}"
dataset_path_to_sample = f"datasets/{args.dataset}/{sample[0][0]}"
# filename_0 = "predictions/{0}/batch_{1}/batch_{1}_sample_{2}.tif".format(prediction_directory_name, batch_index, idy % model_batch_size)
save_img(filename, dataset_path_to_sample, image)
img_0 = array_to_img(image)
# img_0.save(filename.replace(".tif", ".png"))
non_blank_predictions += 1
timeseries_sample = {}
timeseries_sample["timesteps"] = sample
# The name of the prediction produced by this sample
prediction_file_name = f"predictions/{prediction_directory_name}/batch_{idx}/_sample{idy}"
sample_data = {
f"sample_{idy}": timeseries_sample,
"prediction": prediction_file_name
}
samples.append(sample_data)
metadata[f"batch_{idx}"].append(samples)
with open(
'predictions/{0}/{1}_{2}_batch_metadata_{3}.json'.format(
prediction_directory_name, model_name, args.dataset,
current_date_time), 'w') as fp:
json.dump(metadata, fp, indent=4)
print("samples:" + str(len(predictions * model_batch_size)))
# for idx in range(len(test_batch_metadata)):
# os.mkdir("predictions/{0}/batch_{1}".format(prediction_directory_name, idx))
# set to -1 to account for 0 mod 4 = 0 in batch_indexing
# print(len(predictions))
# print(f"Sample Shape: {predictions[0].shape}")
# batch_index = 0
# non_blank_predictions = 0
# for idy, image in enumerate(predictions):
# if idy % model_batch_size == 0 and idy != 0:
# batch_index += 1
# for idz in range(image.shape[-1]):
# images = np.array(image[:, :, idz].reshape(NETWORK_DEMS, NETWORK_DEMS, 1))
# img = images.reshape(NETWORK_DEMS, NETWORK_DEMS, 1).astype(dtype=np.uint8)
# if np.ptp(img) != 0:
# img_0 = array_to_img(img)
# filename_0 = "predictions/{0}/batch_{1}/batch_{1}_sample_{2}.tif".format(prediction_directory_name, batch_index, idy % model_batch_size)
# img_0.save(filename_0)
# non_blank_predictions+=1
print(
f"Total non-blank predictions saved: {non_blank_predictions} out of {len(predictions)} predictions"
)
def print_summary(model_name):
model = load_model(model_name)
model.summary()
from src import app
from src.model import load_model
if __name__ == "__main__":
load_model()
app.run()
