def init_from_config_url(cls, config_path): with open(config_path, "r") as f: config = yaml.load(f, yaml.SafeLoader) download_model( config["model_url"], config["model_path"], config["model_sha256"], ) return cls.init_from_config_path(config_path)
def check_for_ds(): """ Check if Deepspeech is present or not. If not it downloads Italian deepspeech model. :return: """ string = "" # PRINT THE layout = [[sg.Text(size=(100, 5), key='text')]] window = sg.Window('Preparing the game...', layout, finalize=True, size=(300, 200)) window.disappear() # manage the window event, values = window.read(timeout=100) if event == sg.WINDOW_CLOSED or event == 'Quit': exit(0) if not os.path.exists('DS'): os.makedirs('DS') string += "DS" os.chdir("DS") flg_files = utils.check_for_files() flg_xz = utils.check_for_xz() if not flg_xz and not flg_files: window.reappear() window['text'].update("Downloading italian Deepspeech model...") window.refresh() utils.download_model() window['text'].update("Unzipping model...") window.refresh() utils.untar() os.remove("model_tensorflow_it.tar.xz") elif flg_xz: window.reappear() window['text'].update("Unzipping model...") window.refresh() utils.untar() os.remove("model_tensorflow_it.tar.xz") os.chdir("..") window.disappear() window.close()
def initialize_model(): model = download_model(model_name) # normalizacja do sredniej i odchylenia standardowego zbioru ImageNet preprocessing = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], axis=-3) return PyTorchModel(model, bounds=(0, 1), preprocessing=preprocessing)
def __init__(self, model_identifier): assert model_identifier in allowed_model_architectures model_name = _find_model_name(model_identifier) model_alias = _find_model_alias(model_identifier) self._model_fn = nets_factory.get_network_fn(model_alias) self.scalar_mul = nets_factory.get_scalar_mul(model_identifier) self.content_layers = _content_layers_dict[model_alias] self.style_layers = _style_layers_dict[model_alias] model_dir = 'data/lossnet' model_url = model_url_dict[model_identifier] download_model(model_url, model_dir, model_name) self._model_path = os.path.join(model_dir, model_name + '.ckpt') model_alias = _find_model_alias(model_name) self.model_scope = _find_scope_name(model_alias)
y = super(MyModel, self).forward(*x) return y[0] def train_one_batch(self, x, y): pass if __name__ == "__main__": url = 'https://github.com/onnx/models/raw/master/vision/classification/shufflenet/model/shufflenet-v2-10.tar.gz' download_dir = '/tmp/' model_path = os.path.join(download_dir, 'model', 'test_shufflenetv2', 'model.onnx') logging.info("onnx load model...") download_model(url) onnx_model = onnx.load(model_path) # inference logging.info("preprocessing...") img, labels = get_image_labe() img = preprocess(img) # sg_ir = sonnx.prepare(onnx_model) # run without graph # y = sg_ir.run([img]) logging.info("model compling...") dev = device.create_cuda_gpu() x = tensor.PlaceHolder(img.shape, device=dev) model = MyModel(onnx_model) model.compile([x], is_train=False, use_graph=True, sequential=True)
# Log level (if set to debug - debug log level on, esle info level log is set) LOG_LEVEL = os.getenv('LOG_LEVEL') # Logging configuration level_logs = logging.INFO if (LOG_LEVEL == 'debug'): level_logs = logging.DEBUG logging.basicConfig(format='%(asctime)s; %(levelname)s: %(message)s', level=level_logs) # Download model from cloud storage repository MODEL_BUCKET = os.getenv('MODEL_BUCKET') MODEL_NAME = None if MODEL_BUCKET != 'none': MODEL_NAME = os.getenv('MODEL_NAME') download_model(MODEL_BUCKET, MODEL_NAME, MODEL_OUTPUT) else: MODEL_BUCKET = None # Initialize training controller training_controller = TrainingController(MODEL_BUCKET, MODEL_NAME) # Run specyfic functionality based on global command if CMD == 'train': SMOOTHING_FACTOR = float(os.getenv('SMOOTHING_FACTOR')) TRAINING_SET_PATH = os.getenv('TRAINING_SET_PATH') if not os.path.exists(TRAINING_SET_PATH): os.makedirs(TRAINING_SET_PATH) EPOCH = int(os.getenv('EPOCH')) BATCH_SIZE = int(os.getenv('BATCH_SIZE')) SAVE_INTERVAL = int(os.getenv('SAVE_INTERVAL'))
def setup(session): model_url = utils.download_model('en-ner-person.bin', session) sent_url = utils.download_model('en-sent.bin', session) search_context = { "name": search_context_name, "matchers": [{ "name": "EmailMatcher", "type": "pattern", "pattern": r"\b[\w._%+-]+@[\w.-]+\.[A-Za-z]{2,4}\b" }, { "name": "PhoneMatcher", "type": "pattern", "pattern": r"\b(\+?1?([ .-]?)?)?(\(?([2-9]\d{2})\)?([ .-]?)?)([2-9]\d{2})([ .-]?)(\d{4})(?: #?[eE][xX][tT]\.? \d{2,6})?\b" }, { "name": "NameMatcher", "type": "ner", "modelUrl": model_url, "sentenceDetectorUrl": sent_url }] } mask_context = { "name": mask_context_name, "rules": [{ "name": "HashRule", "type": "cosort", "expression": "hash_sha2($\{INPUT\})" }, { "name": "FpeRule", "type": "cosort", "expression": "enc_fp_aes256_alphanum($\{INPUT\})" }], "ruleMatchers": [{ "name": "FpeRuleMatcher", "type": "name", "rule": "FpeRule", "pattern": "NameMatcher|PhoneMatcher" }, { "name": "HashRuleMatcher", "type": "name", "rule": "HashRule", "pattern": "EmailMatcher" }] } file_search_context = { "name": file_search_context_name, "matchers": [{ "name": search_context_name, "type": "searchContext" }, { "name": "NameMatcher", "type": "jsonPath", "jsonPath": "$..name" }, { "name": "NameMatcher", "type": "xmlPath", "xmlPath": "//name" }] } file_mask_context = { "name": file_mask_context_name, "rules": [{ "name": mask_context_name, "type": "maskContext" }] } utils.create_context("searchContext", search_context, session) utils.create_context("maskContext", mask_context, session) utils.create_context("files/fileSearchContext", file_search_context, session) utils.create_context("files/fileMaskContext", file_mask_context, session)
def save_heatmap(model_name, path, destination_path): image = Image.open(path) # klasa potrzebana do ustawienia hooka na ostatnia warstwe konwolucjna class Hook(): features = None def __init__(self, layer): self.hook = layer.register_forward_hook(self.get_activation_map) def get_activation_map(self, module, input, output): self.activation_maps = ((output.cpu()).data).numpy() def remove(self): self.hook.remove() # inicjalizacja modelu model = download_model(model_name) if "resnet" in model_name: last_convolutional_layer_name = list(model._modules.keys())[-3] classification_layer_name = list(model._modules.keys())[-1] else: last_convolutional_layer_name = list(model._modules.keys())[-2] classification_layer_name = list(model._modules.keys())[-1] # definicja normalizacji do sredniej i odchylenia standardowego zbioru ImageNet normalization = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) # definicja preprocessingu - zmiana rozmiaru, konwersja na tensor i normalizacja preprocessing = transforms.Compose( [transforms.Resize((224, 224)), transforms.ToTensor(), normalization]) # transformacja potrzebna do wyswietlenia zdjecia reshaping = transforms.Compose([transforms.Resize((224, 224))]) # preprocessing zdjecia tensor = preprocessing(image) # ustawienie hooka na ostatnia warstwe konwolucyjna last_convolutional_layer = model._modules.get( last_convolutional_layer_name) hook = Hook(last_convolutional_layer) # podanie zdjecia na wejscie modelu predictions = model(Variable((tensor.unsqueeze(0)), requires_grad=True)) hook.remove() # podanie predykcji na funkcjÄ™ softmax w celu uzyskania prawdopodobienstw probabilities = softmax(predictions).data.squeeze() # pobranie parametrow warstwy klasyfikacyjnej classification_layer = list( model._modules.get(classification_layer_name).parameters()) classification_layer = np.squeeze( classification_layer[0].cpu().data.numpy()) # okreslenie klasy o najwyzszyej pewnosci predykcji predicted_class = topk(probabilities, 1)[1].int() # stworzenie mapy ciepla aktywacji klas heatmap = get_heatmap(hook.activation_maps, classification_layer, predicted_class) # zapis zdjecia z nalozona mapa ciepla aktywacji klas plt.imshow(reshaping(image)) plt.imshow(skimage.transform.resize(heatmap[0], tensor.shape[1:3]), alpha=0.5, cmap='jet') plt.axis('off') plt.savefig(destination_path, bbox_inches='tight')
config.hvd = hvd hvd.init() np.random.seed(args.seed) torch.manual_seed(args.seed) torch.cuda.manual_seed_all(args.seed) torch.backends.cudnn.deterministic = True torch.cuda.set_device(hvd.local_rank()) config.rank = hvd.rank() config.world = hvd.size() if hvd.local_rank() == 0: utils.download_model(config) hvd.broadcast_object(0, root_rank=0) model = x.Model(config) start_time = time.time() print('Loading dataset') train_data, dev_data, test_data = utils.build_dataset(config) train_iter = utils.build_dataloader(train_data, config) dev_iter = utils.build_dataloader(dev_data, config) test_iter = utils.build_dataloader(test_data, config) time_dif = utils.get_time_dif(start_time) print("Prepare data time: ", time_dif) # Train, eval, test
def setup(session): model_url = utils.download_model('en-ner-person.bin',session) sent_url = utils.download_model('en-sent.bin',session) token_url = utils.download_model('en-token.bin',session) search_context = { "name": search_context_name, "matchers": [ { "name": "EmailMatcher", "type": "pattern", "pattern": r"\b[\w._%+-]+@[\w.-]+\.[A-Za-z]{2,4}\b" }, ] } search_context_ner = { "name": search_context_ner_name, "matchers": [ { "name": "NameMatcher", "type": "ner", "modelUrl": model_url, "sentenceDetectorUrl": sent_url, "tokenizerUrl": token_url } ] } mask_context = { "name": mask_context_name, "rules": [ { "name": "HashEmailRule", "type": "cosort", "expression": r"hash_sha2(${EMAIL})" }, { "name": "FpeNameRule", "type": "cosort", "expression": r"enc_fp_aes256_alphanum(${NAME},'passphrase')" } ], "ruleMatchers": [ { "name": "HashRuleMatcher", "type": "name", "rule": "HashEmailRule", "pattern": "EmailMatcher" }, { "name": "NameRuleMatcher", "type": "name", "rule": "FpeNameRule", "pattern": "NameMatcher" } ] } file_search_context = { "name": file_search_context_name, "matchers": [ { "name": search_context_name, "type": "searchContext" }, { "name": search_context_ner_name, "type": "searchContext", "contentFilters": { "columns": [ { "ignoreHeader": True, "pattern": "comment" } ] } }, { "name": "NameMatcher", "type": "column", "ignoreHeader": True, "pattern": ".*name" } ] } file_mask_context = { "name": file_mask_context_name, "rules": [ { "name": mask_context_name, "type": "maskContext" } ] } utils.create_context("searchContext", search_context,session) utils.create_context("searchContext", search_context_ner,session) utils.create_context("maskContext", mask_context,session) utils.create_context("files/fileSearchContext", file_search_context,session) utils.create_context("files/fileMaskContext", file_mask_context,session)
def run(global_rank, world_size, local_rank, max_epoch, batch_size, model_config, data, sgd, graph, verbosity, dist_option='fp32', spars=None): dev = device.create_cuda_gpu_on(local_rank) dev.SetRandSeed(0) np.random.seed(0) if data == 'cifar10': from data import cifar10 train_x, train_y, val_x, val_y = cifar10.load() elif data == 'cifar100': from data import cifar100 train_x, train_y, val_x, val_y = cifar100.load() num_channels = train_x.shape[1] image_size = train_x.shape[2] data_size = np.prod(train_x.shape[1:train_x.ndim]).item() num_classes = (np.max(train_y) + 1).item() # read and make onnx model download_model(model_config['url']) onnx_model = onnx.load(os.path.join('/tmp', model_config['path'])) model = MyModel(onnx_model, num_channels=num_channels, num_classes=num_classes, last_layers=model_config['last_layers'], in_dim=model_config['last_layers_dim']) # For distributed training, sequential gives better performance if hasattr(sgd, "communicator"): DIST = True sequential = True else: DIST = False sequential = False if DIST: train_x, train_y, val_x, val_y = partition(global_rank, world_size, train_x, train_y, val_x, val_y) ''' # check dataset shape correctness if global_rank == 0: print("Check the shape of dataset:") print(train_x.shape) print(train_y.shape) ''' if model.dimension == 4: tx = tensor.Tensor( (batch_size, num_channels, model.input_size, model.input_size), dev, tensor.float32) elif model.dimension == 2: tx = tensor.Tensor((batch_size, data_size), dev, tensor.float32) np.reshape(train_x, (train_x.shape[0], -1)) np.reshape(val_x, (val_x.shape[0], -1)) ty = tensor.Tensor((batch_size, ), dev, tensor.int32) num_train_batch = train_x.shape[0] // batch_size num_val_batch = val_x.shape[0] // batch_size idx = np.arange(train_x.shape[0], dtype=np.int32) # attached model to graph model.set_optimizer(sgd) model.compile([tx], is_train=True, use_graph=graph, sequential=sequential) dev.SetVerbosity(verbosity) # Training and Evaluation Loop for epoch in range(max_epoch): start_time = time.time() np.random.shuffle(idx) if global_rank == 0: print('Starting Epoch %d:' % (epoch)) # Training Phase train_correct = np.zeros(shape=[1], dtype=np.float32) test_correct = np.zeros(shape=[1], dtype=np.float32) train_loss = np.zeros(shape=[1], dtype=np.float32) model.train() for b in tqdm(range(num_train_batch)): # Generate the patch data in this iteration x = train_x[idx[b * batch_size:(b + 1) * batch_size]] if model.dimension == 4: x = augmentation(x, batch_size) if (image_size != model.input_size): x = resize_dataset(x, model.input_size) y = train_y[idx[b * batch_size:(b + 1) * batch_size]] # Copy the patch data into input tensors tx.copy_from_numpy(x) ty.copy_from_numpy(y) # Train the model out, loss = model(tx, ty, dist_option, spars) train_correct += accuracy(tensor.to_numpy(out), y) train_loss += tensor.to_numpy(loss)[0] if DIST: # Reduce the Evaluation Accuracy and Loss from Multiple Devices reducer = tensor.Tensor((1, ), dev, tensor.float32) train_correct = reduce_variable(train_correct, sgd, reducer) train_loss = reduce_variable(train_loss, sgd, reducer) if global_rank == 0: print('Training loss = %f, training accuracy = %f' % (train_loss, train_correct / (num_train_batch * batch_size * world_size)), flush=True) # Evaluation Phase model.eval() for b in tqdm(range(num_val_batch)): x = val_x[b * batch_size:(b + 1) * batch_size] if model.dimension == 4: if (image_size != model.input_size): x = resize_dataset(x, model.input_size) y = val_y[b * batch_size:(b + 1) * batch_size] tx.copy_from_numpy(x) ty.copy_from_numpy(y) out_test = model(tx) test_correct += accuracy(tensor.to_numpy(out_test), y) if DIST: # Reduce the Evaulation Accuracy from Multiple Devices test_correct = reduce_variable(test_correct, sgd, reducer) # Output the Evaluation Accuracy if global_rank == 0: print('Evaluation accuracy = %f, Elapsed Time = %fs' % (test_correct / (num_val_batch * batch_size * world_size), time.time() - start_time), flush=True) dev.PrintTimeProfiling()
def main(): """ main function to run the style transfer. """ parser = argparse.ArgumentParser() parser.add_argument("-s", "--style", type=str, help="style image") parser.add_argument("-c", "--content", type=str, help="content image") parser.add_argument("--height", type=int, default=255, help="image height") parser.add_argument("--width", type=int, default=333, help="image width") parser.add_argument("-nr", "--noise_ratio", type=int, default=0.5, help="noise ratio when combing images") args = parser.parse_args() CONTENT_IMAGE = args.content STYLE_IMAGE = args.style IMAGE_HEIGHT = args.height IMAGE_WIDTH = args.width NR = args.noise_ratio with tf.variable_scope("input"): # Define input image as variable, so we can directly # modify it to get the output image input_image = tf.Variable(np.zeros([1, IMAGE_HEIGHT, IMAGE_WIDTH, 3]), dtype=np.float32) # Download the model and make new directories utils.download_model(VGG_DOWNLOAD_LINK, VGG_MODEL, EXPECTED_BYTES) utils.make_dir("./checkpoints") utils.make_dir("./graphs") utils.make_dir("./outputs") # Initialize the model with given input image vgg = vgg_model.VGG_loader(file_path=VGG_MODEL) model = vgg.load_image(input_image) # Initialize the global step model["global_step"] = tf.Variable(0, dtype=tf.int32, trainable=False, name="global_step") content_image = utils.get_resized_image(CONTENT_IMAGE, IMAGE_HEIGHT, IMAGE_WIDTH) content_image -= MEAN_PIXELS style_image = utils.get_resized_image(STYLE_IMAGE, IMAGE_HEIGHT, IMAGE_WIDTH) style_image -= MEAN_PIXELS model["content_loss"], model["style_loss"], model["total_loss"] = ( _create_losses(model, input_image, content_image, style_image)) # Define the optimizer model["optimizer"] = (tf.train.AdamOptimizer(LR).minimize( model["total_loss"])) # Define the summary op model["summary_op"] = _create_summary(model) # Generate initial image initial_image = (utils.generate_noise_image(content_image, IMAGE_HEIGHT, IMAGE_WIDTH, NR)) # Finally, run the optimizer to get the style transferred image train(model, input_image, initial_image)
def setup(session): model_url = utils.download_model('en-ner-person.bin',session) sent_url = utils.download_model('en-sent.bin',session) token_url = utils.download_model('en-token.bin',session) search_context = { "name": search_context_name, "matchers": [ { "name": "SsnMatcher", "type": "pattern", "pattern": r"\b(\d{3}[-]?\d{2}[-]?\d{4})\b" }, { "name": "EmailMatcher", "type": "pattern", "pattern": r"\b[\w._%+-]+@[\w.-]+\.[A-Za-z]{2,4}\b" }, { "name": "PhoneMatcher", "type": "pattern", "pattern": r"\b(\+?1?([ .-]?)?)?(\(?([2-9]\d{2})\)?([ .-]?)?)([2-9]\d{2})([ .-]?)(\d{4})(?: #?[eE][xX][tT]\.? \d{2,6})?\b" }, { "name": "NameMatcher", "type": "ner", "modelUrl": model_url, "sentenceDetectorUrl": sent_url, "tokenizerUrl": token_url } ] } mask_context = { "name": mask_context_name, "rules": [ { "name": "HashEmailRule", "type": "cosort", "expression": r"hash_sha2(${EMAIL})" }, { "name": "FpeRule", "type": "cosort", "expression": r"enc_fp_aes256_alphanum(${NAME})" }, { "name": "RedactSsnRule", "type": "cosort", "expression": r"replace_chars(${SSN},'*',1,3,'*',5,2)" } ], "ruleMatchers": [ { "name": "FpeRuleMatcher", "type": "name", "rule": "FpeRule", "pattern": "NameMatcher|PhoneMatcher" }, { "name": "EmailRuleMatcher", "type": "name", "rule": "HashEmailRule", "pattern": "EmailMatcher" }, { "name": "SsnRuleMatcher", "type": "name", "rule": "RedactSsnRule", "pattern": "SsnMatcher" } ] } file_search_context = { "name": file_search_context_name, "matchers": [ { "name": search_context_name, "type": "searchContext" } ] } file_mask_context = { "name": file_mask_context_name, "rules": [ { "name": mask_context_name, "type": "maskContext" } ] } utils.create_context("searchContext", search_context,session) utils.create_context("maskContext", mask_context,session) utils.create_context("files/fileSearchContext", file_search_context,session) utils.create_context("files/fileMaskContext", file_mask_context,session)