def init_from_config_url(cls, config_path):
with open(config_path, "r") as f:
config = yaml.load(f, yaml.SafeLoader)
download_model(
config["model_url"],
config["model_path"],
config["model_sha256"],
)
return cls.init_from_config_path(config_path)
def check_for_ds():
"""
Check if Deepspeech is present or not. If not it downloads Italian deepspeech model.
:return:
"""
string = ""
# PRINT THE
layout = [[sg.Text(size=(100, 5), key='text')]]
window = sg.Window('Preparing the game...',
layout,
finalize=True,
size=(300, 200))
window.disappear()
# manage the window
event, values = window.read(timeout=100)
if event == sg.WINDOW_CLOSED or event == 'Quit':
exit(0)
if not os.path.exists('DS'):
os.makedirs('DS')
string += "DS"
os.chdir("DS")
flg_files = utils.check_for_files()
flg_xz = utils.check_for_xz()
if not flg_xz and not flg_files:
window.reappear()
window['text'].update("Downloading italian Deepspeech model...")
window.refresh()
utils.download_model()
window['text'].update("Unzipping model...")
window.refresh()
utils.untar()
os.remove("model_tensorflow_it.tar.xz")
elif flg_xz:
window.reappear()
window['text'].update("Unzipping model...")
window.refresh()
utils.untar()
os.remove("model_tensorflow_it.tar.xz")
os.chdir("..")
window.disappear()
window.close()
def initialize_model():
model = download_model(model_name)
# normalizacja do sredniej i odchylenia standardowego zbioru ImageNet
preprocessing = dict(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
axis=-3)
return PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
def __init__(self, model_identifier):
assert model_identifier in allowed_model_architectures
model_name = _find_model_name(model_identifier)
model_alias = _find_model_alias(model_identifier)
self._model_fn = nets_factory.get_network_fn(model_alias)
self.scalar_mul = nets_factory.get_scalar_mul(model_identifier)
self.content_layers = _content_layers_dict[model_alias]
self.style_layers = _style_layers_dict[model_alias]
model_dir = 'data/lossnet'
model_url = model_url_dict[model_identifier]
download_model(model_url, model_dir, model_name)
self._model_path = os.path.join(model_dir, model_name + '.ckpt')
model_alias = _find_model_alias(model_name)
self.model_scope = _find_scope_name(model_alias)
y = super(MyModel, self).forward(*x)
return y[0]
def train_one_batch(self, x, y):
pass
if __name__ == "__main__":
url = 'https://github.com/onnx/models/raw/master/vision/classification/shufflenet/model/shufflenet-v2-10.tar.gz'
download_dir = '/tmp/'
model_path = os.path.join(download_dir, 'model', 'test_shufflenetv2',
'model.onnx')
logging.info("onnx load model...")
download_model(url)
onnx_model = onnx.load(model_path)
# inference
logging.info("preprocessing...")
img, labels = get_image_labe()
img = preprocess(img)
# sg_ir = sonnx.prepare(onnx_model) # run without graph
# y = sg_ir.run([img])
logging.info("model compling...")
dev = device.create_cuda_gpu()
x = tensor.PlaceHolder(img.shape, device=dev)
model = MyModel(onnx_model)
model.compile([x], is_train=False, use_graph=True, sequential=True)
# Log level (if set to debug - debug log level on, esle info level log is set)
LOG_LEVEL = os.getenv('LOG_LEVEL')
# Logging configuration
level_logs = logging.INFO
if (LOG_LEVEL == 'debug'):
level_logs = logging.DEBUG
logging.basicConfig(format='%(asctime)s; %(levelname)s: %(message)s',
level=level_logs)
# Download model from cloud storage repository
MODEL_BUCKET = os.getenv('MODEL_BUCKET')
MODEL_NAME = None
if MODEL_BUCKET != 'none':
MODEL_NAME = os.getenv('MODEL_NAME')
download_model(MODEL_BUCKET, MODEL_NAME, MODEL_OUTPUT)
else:
MODEL_BUCKET = None
# Initialize training controller
training_controller = TrainingController(MODEL_BUCKET, MODEL_NAME)
# Run specyfic functionality based on global command
if CMD == 'train':
SMOOTHING_FACTOR = float(os.getenv('SMOOTHING_FACTOR'))
TRAINING_SET_PATH = os.getenv('TRAINING_SET_PATH')
if not os.path.exists(TRAINING_SET_PATH):
os.makedirs(TRAINING_SET_PATH)
EPOCH = int(os.getenv('EPOCH'))
BATCH_SIZE = int(os.getenv('BATCH_SIZE'))
SAVE_INTERVAL = int(os.getenv('SAVE_INTERVAL'))
def setup(session):
model_url = utils.download_model('en-ner-person.bin', session)
sent_url = utils.download_model('en-sent.bin', session)
search_context = {
"name":
search_context_name,
"matchers": [{
"name": "EmailMatcher",
"type": "pattern",
"pattern": r"\b[\w._%+-]+@[\w.-]+\.[A-Za-z]{2,4}\b"
}, {
"name":
"PhoneMatcher",
"type":
"pattern",
"pattern":
r"\b(\+?1?([ .-]?)?)?(\(?([2-9]\d{2})\)?([ .-]?)?)([2-9]\d{2})([ .-]?)(\d{4})(?: #?[eE][xX][tT]\.? \d{2,6})?\b"
}, {
"name": "NameMatcher",
"type": "ner",
"modelUrl": model_url,
"sentenceDetectorUrl": sent_url
}]
}
mask_context = {
"name":
mask_context_name,
"rules": [{
"name": "HashRule",
"type": "cosort",
"expression": "hash_sha2($\{INPUT\})"
}, {
"name": "FpeRule",
"type": "cosort",
"expression": "enc_fp_aes256_alphanum($\{INPUT\})"
}],
"ruleMatchers": [{
"name": "FpeRuleMatcher",
"type": "name",
"rule": "FpeRule",
"pattern": "NameMatcher|PhoneMatcher"
}, {
"name": "HashRuleMatcher",
"type": "name",
"rule": "HashRule",
"pattern": "EmailMatcher"
}]
}
file_search_context = {
"name":
file_search_context_name,
"matchers": [{
"name": search_context_name,
"type": "searchContext"
}, {
"name": "NameMatcher",
"type": "jsonPath",
"jsonPath": "$..name"
}, {
"name": "NameMatcher",
"type": "xmlPath",
"xmlPath": "//name"
}]
}
file_mask_context = {
"name": file_mask_context_name,
"rules": [{
"name": mask_context_name,
"type": "maskContext"
}]
}
utils.create_context("searchContext", search_context, session)
utils.create_context("maskContext", mask_context, session)
utils.create_context("files/fileSearchContext", file_search_context,
session)
utils.create_context("files/fileMaskContext", file_mask_context, session)
def save_heatmap(model_name, path, destination_path):
image = Image.open(path)
# klasa potrzebana do ustawienia hooka na ostatnia warstwe konwolucjna
class Hook():
features = None
def __init__(self, layer):
self.hook = layer.register_forward_hook(self.get_activation_map)
def get_activation_map(self, module, input, output):
self.activation_maps = ((output.cpu()).data).numpy()
def remove(self):
self.hook.remove()
# inicjalizacja modelu
model = download_model(model_name)
if "resnet" in model_name:
last_convolutional_layer_name = list(model._modules.keys())[-3]
classification_layer_name = list(model._modules.keys())[-1]
else:
last_convolutional_layer_name = list(model._modules.keys())[-2]
classification_layer_name = list(model._modules.keys())[-1]
# definicja normalizacji do sredniej i odchylenia standardowego zbioru ImageNet
normalization = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
# definicja preprocessingu - zmiana rozmiaru, konwersja na tensor i normalizacja
preprocessing = transforms.Compose(
[transforms.Resize((224, 224)),
transforms.ToTensor(), normalization])
# transformacja potrzebna do wyswietlenia zdjecia
reshaping = transforms.Compose([transforms.Resize((224, 224))])
# preprocessing zdjecia
tensor = preprocessing(image)
# ustawienie hooka na ostatnia warstwe konwolucyjna
last_convolutional_layer = model._modules.get(
last_convolutional_layer_name)
hook = Hook(last_convolutional_layer)
# podanie zdjecia na wejscie modelu
predictions = model(Variable((tensor.unsqueeze(0)), requires_grad=True))
hook.remove()
# podanie predykcji na funkcję softmax w celu uzyskania prawdopodobienstw
probabilities = softmax(predictions).data.squeeze()
# pobranie parametrow warstwy klasyfikacyjnej
classification_layer = list(
model._modules.get(classification_layer_name).parameters())
classification_layer = np.squeeze(
classification_layer[0].cpu().data.numpy())
# okreslenie klasy o najwyzszyej pewnosci predykcji
predicted_class = topk(probabilities, 1)[1].int()
# stworzenie mapy ciepla aktywacji klas
heatmap = get_heatmap(hook.activation_maps, classification_layer,
predicted_class)
# zapis zdjecia z nalozona mapa ciepla aktywacji klas
plt.imshow(reshaping(image))
plt.imshow(skimage.transform.resize(heatmap[0], tensor.shape[1:3]),
alpha=0.5,
cmap='jet')
plt.axis('off')
plt.savefig(destination_path, bbox_inches='tight')
config.hvd = hvd
hvd.init()
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
torch.backends.cudnn.deterministic = True
torch.cuda.set_device(hvd.local_rank())
config.rank = hvd.rank()
config.world = hvd.size()
if hvd.local_rank() == 0:
utils.download_model(config)
hvd.broadcast_object(0, root_rank=0)
model = x.Model(config)
start_time = time.time()
print('Loading dataset')
train_data, dev_data, test_data = utils.build_dataset(config)
train_iter = utils.build_dataloader(train_data, config)
dev_iter = utils.build_dataloader(dev_data, config)
test_iter = utils.build_dataloader(test_data, config)
time_dif = utils.get_time_dif(start_time)
print("Prepare data time: ", time_dif)
# Train, eval, test
def setup(session):
model_url = utils.download_model('en-ner-person.bin',session)
sent_url = utils.download_model('en-sent.bin',session)
token_url = utils.download_model('en-token.bin',session)
search_context = {
"name": search_context_name,
"matchers": [
{
"name": "EmailMatcher",
"type": "pattern",
"pattern": r"\b[\w._%+-]+@[\w.-]+\.[A-Za-z]{2,4}\b"
},
]
}
search_context_ner = {
"name": search_context_ner_name,
"matchers": [
{
"name": "NameMatcher",
"type": "ner",
"modelUrl": model_url,
"sentenceDetectorUrl": sent_url,
"tokenizerUrl": token_url
}
]
}
mask_context = {
"name": mask_context_name,
"rules": [
{
"name": "HashEmailRule",
"type": "cosort",
"expression": r"hash_sha2(${EMAIL})"
},
{
"name": "FpeNameRule",
"type": "cosort",
"expression": r"enc_fp_aes256_alphanum(${NAME},'passphrase')"
}
],
"ruleMatchers": [
{
"name": "HashRuleMatcher",
"type": "name",
"rule": "HashEmailRule",
"pattern": "EmailMatcher"
},
{
"name": "NameRuleMatcher",
"type": "name",
"rule": "FpeNameRule",
"pattern": "NameMatcher"
}
]
}
file_search_context = {
"name": file_search_context_name,
"matchers": [
{
"name": search_context_name,
"type": "searchContext"
},
{
"name": search_context_ner_name,
"type": "searchContext",
"contentFilters": {
"columns": [
{
"ignoreHeader": True,
"pattern": "comment"
}
]
}
},
{
"name": "NameMatcher",
"type": "column",
"ignoreHeader": True,
"pattern": ".*name"
}
]
}
file_mask_context = {
"name": file_mask_context_name,
"rules": [
{
"name": mask_context_name,
"type": "maskContext"
}
]
}
utils.create_context("searchContext", search_context,session)
utils.create_context("searchContext", search_context_ner,session)
utils.create_context("maskContext", mask_context,session)
utils.create_context("files/fileSearchContext", file_search_context,session)
utils.create_context("files/fileMaskContext", file_mask_context,session)
def run(global_rank,
world_size,
local_rank,
max_epoch,
batch_size,
model_config,
data,
sgd,
graph,
verbosity,
dist_option='fp32',
spars=None):
dev = device.create_cuda_gpu_on(local_rank)
dev.SetRandSeed(0)
np.random.seed(0)
if data == 'cifar10':
from data import cifar10
train_x, train_y, val_x, val_y = cifar10.load()
elif data == 'cifar100':
from data import cifar100
train_x, train_y, val_x, val_y = cifar100.load()
num_channels = train_x.shape[1]
image_size = train_x.shape[2]
data_size = np.prod(train_x.shape[1:train_x.ndim]).item()
num_classes = (np.max(train_y) + 1).item()
# read and make onnx model
download_model(model_config['url'])
onnx_model = onnx.load(os.path.join('/tmp', model_config['path']))
model = MyModel(onnx_model,
num_channels=num_channels,
num_classes=num_classes,
last_layers=model_config['last_layers'],
in_dim=model_config['last_layers_dim'])
# For distributed training, sequential gives better performance
if hasattr(sgd, "communicator"):
DIST = True
sequential = True
else:
DIST = False
sequential = False
if DIST:
train_x, train_y, val_x, val_y = partition(global_rank, world_size,
train_x, train_y, val_x,
val_y)
'''
# check dataset shape correctness
if global_rank == 0:
print("Check the shape of dataset:")
print(train_x.shape)
print(train_y.shape)
'''
if model.dimension == 4:
tx = tensor.Tensor(
(batch_size, num_channels, model.input_size, model.input_size),
dev, tensor.float32)
elif model.dimension == 2:
tx = tensor.Tensor((batch_size, data_size), dev, tensor.float32)
np.reshape(train_x, (train_x.shape[0], -1))
np.reshape(val_x, (val_x.shape[0], -1))
ty = tensor.Tensor((batch_size, ), dev, tensor.int32)
num_train_batch = train_x.shape[0] // batch_size
num_val_batch = val_x.shape[0] // batch_size
idx = np.arange(train_x.shape[0], dtype=np.int32)
# attached model to graph
model.set_optimizer(sgd)
model.compile([tx], is_train=True, use_graph=graph, sequential=sequential)
dev.SetVerbosity(verbosity)
# Training and Evaluation Loop
for epoch in range(max_epoch):
start_time = time.time()
np.random.shuffle(idx)
if global_rank == 0:
print('Starting Epoch %d:' % (epoch))
# Training Phase
train_correct = np.zeros(shape=[1], dtype=np.float32)
test_correct = np.zeros(shape=[1], dtype=np.float32)
train_loss = np.zeros(shape=[1], dtype=np.float32)
model.train()
for b in tqdm(range(num_train_batch)):
# Generate the patch data in this iteration
x = train_x[idx[b * batch_size:(b + 1) * batch_size]]
if model.dimension == 4:
x = augmentation(x, batch_size)
if (image_size != model.input_size):
x = resize_dataset(x, model.input_size)
y = train_y[idx[b * batch_size:(b + 1) * batch_size]]
# Copy the patch data into input tensors
tx.copy_from_numpy(x)
ty.copy_from_numpy(y)
# Train the model
out, loss = model(tx, ty, dist_option, spars)
train_correct += accuracy(tensor.to_numpy(out), y)
train_loss += tensor.to_numpy(loss)[0]
if DIST:
# Reduce the Evaluation Accuracy and Loss from Multiple Devices
reducer = tensor.Tensor((1, ), dev, tensor.float32)
train_correct = reduce_variable(train_correct, sgd, reducer)
train_loss = reduce_variable(train_loss, sgd, reducer)
if global_rank == 0:
print('Training loss = %f, training accuracy = %f' %
(train_loss, train_correct /
(num_train_batch * batch_size * world_size)),
flush=True)
# Evaluation Phase
model.eval()
for b in tqdm(range(num_val_batch)):
x = val_x[b * batch_size:(b + 1) * batch_size]
if model.dimension == 4:
if (image_size != model.input_size):
x = resize_dataset(x, model.input_size)
y = val_y[b * batch_size:(b + 1) * batch_size]
tx.copy_from_numpy(x)
ty.copy_from_numpy(y)
out_test = model(tx)
test_correct += accuracy(tensor.to_numpy(out_test), y)
if DIST:
# Reduce the Evaulation Accuracy from Multiple Devices
test_correct = reduce_variable(test_correct, sgd, reducer)
# Output the Evaluation Accuracy
if global_rank == 0:
print('Evaluation accuracy = %f, Elapsed Time = %fs' %
(test_correct / (num_val_batch * batch_size * world_size),
time.time() - start_time),
flush=True)
dev.PrintTimeProfiling()
def main():
""" main function to run the style transfer.
"""
parser = argparse.ArgumentParser()
parser.add_argument("-s", "--style", type=str, help="style image")
parser.add_argument("-c", "--content", type=str, help="content image")
parser.add_argument("--height", type=int, default=255, help="image height")
parser.add_argument("--width", type=int, default=333, help="image width")
parser.add_argument("-nr",
"--noise_ratio",
type=int,
default=0.5,
help="noise ratio when combing images")
args = parser.parse_args()
CONTENT_IMAGE = args.content
STYLE_IMAGE = args.style
IMAGE_HEIGHT = args.height
IMAGE_WIDTH = args.width
NR = args.noise_ratio
with tf.variable_scope("input"):
# Define input image as variable, so we can directly
# modify it to get the output image
input_image = tf.Variable(np.zeros([1, IMAGE_HEIGHT, IMAGE_WIDTH, 3]),
dtype=np.float32)
# Download the model and make new directories
utils.download_model(VGG_DOWNLOAD_LINK, VGG_MODEL, EXPECTED_BYTES)
utils.make_dir("./checkpoints")
utils.make_dir("./graphs")
utils.make_dir("./outputs")
# Initialize the model with given input image
vgg = vgg_model.VGG_loader(file_path=VGG_MODEL)
model = vgg.load_image(input_image)
# Initialize the global step
model["global_step"] = tf.Variable(0,
dtype=tf.int32,
trainable=False,
name="global_step")
content_image = utils.get_resized_image(CONTENT_IMAGE, IMAGE_HEIGHT,
IMAGE_WIDTH)
content_image -= MEAN_PIXELS
style_image = utils.get_resized_image(STYLE_IMAGE, IMAGE_HEIGHT,
IMAGE_WIDTH)
style_image -= MEAN_PIXELS
model["content_loss"], model["style_loss"], model["total_loss"] = (
_create_losses(model, input_image, content_image, style_image))
# Define the optimizer
model["optimizer"] = (tf.train.AdamOptimizer(LR).minimize(
model["total_loss"]))
# Define the summary op
model["summary_op"] = _create_summary(model)
# Generate initial image
initial_image = (utils.generate_noise_image(content_image, IMAGE_HEIGHT,
IMAGE_WIDTH, NR))
# Finally, run the optimizer to get the style transferred image
train(model, input_image, initial_image)
def setup(session):
model_url = utils.download_model('en-ner-person.bin',session)
sent_url = utils.download_model('en-sent.bin',session)
token_url = utils.download_model('en-token.bin',session)
search_context = {
"name": search_context_name,
"matchers": [
{
"name": "SsnMatcher",
"type": "pattern",
"pattern": r"\b(\d{3}[-]?\d{2}[-]?\d{4})\b"
},
{
"name": "EmailMatcher",
"type": "pattern",
"pattern": r"\b[\w._%+-]+@[\w.-]+\.[A-Za-z]{2,4}\b"
},
{
"name": "PhoneMatcher",
"type": "pattern",
"pattern": r"\b(\+?1?([ .-]?)?)?(\(?([2-9]\d{2})\)?([ .-]?)?)([2-9]\d{2})([ .-]?)(\d{4})(?: #?[eE][xX][tT]\.? \d{2,6})?\b"
},
{
"name": "NameMatcher",
"type": "ner",
"modelUrl": model_url,
"sentenceDetectorUrl": sent_url,
"tokenizerUrl": token_url
}
]
}
mask_context = {
"name": mask_context_name,
"rules": [
{
"name": "HashEmailRule",
"type": "cosort",
"expression": r"hash_sha2(${EMAIL})"
},
{
"name": "FpeRule",
"type": "cosort",
"expression": r"enc_fp_aes256_alphanum(${NAME})"
},
{
"name": "RedactSsnRule",
"type": "cosort",
"expression": r"replace_chars(${SSN},'*',1,3,'*',5,2)"
}
],
"ruleMatchers": [
{
"name": "FpeRuleMatcher",
"type": "name",
"rule": "FpeRule",
"pattern": "NameMatcher|PhoneMatcher"
},
{
"name": "EmailRuleMatcher",
"type": "name",
"rule": "HashEmailRule",
"pattern": "EmailMatcher"
},
{
"name": "SsnRuleMatcher",
"type": "name",
"rule": "RedactSsnRule",
"pattern": "SsnMatcher"
}
]
}
file_search_context = {
"name": file_search_context_name,
"matchers": [
{
"name": search_context_name,
"type": "searchContext"
}
]
}
file_mask_context = {
"name": file_mask_context_name,
"rules": [
{
"name": mask_context_name,
"type": "maskContext"
}
]
}
utils.create_context("searchContext", search_context,session)
utils.create_context("maskContext", mask_context,session)
utils.create_context("files/fileSearchContext", file_search_context,session)
utils.create_context("files/fileMaskContext", file_mask_context,session)
