def Classify(filename): with CustomObjectScope({'softmax_v2': tf.keras.activations.softmax}): model = tf.keras.models.load_model('network.h5') img = mpimg.imread(filename) prediction = model.predict(img.reshape(1, 28, 28, 1)) return prediction.argmax()
def load_model_weight(path): with CustomObjectScope({ 'dice_loss': dice_loss, 'dice_coef': dice_coef, 'bce_dice_loss': bce_dice_loss, 'iou': iou }): model = load_model(path) return model
def load_model(self, data): self._model_data = Settings() self._model_data.one_hot_mappings = data["one_hot_mappings"] self._init_extractors() with CustomObjectScope({'GlorotUniform': glorot_uniform()}): model = model_from_json(data["model_json"]) model.set_weights(list(map(lambda l: np.array(l, dtype=float), data["model_weights"]))) self._trained_model = model
def __init__(self, resource_path, conf): """ Parameters ---------- resource_path: String. Raw data path. conf: Dictionary. Configuration. """ # Initialize. self.resource_path = resource_path self.conf = conf self.hps = conf['hps'] self.nn_arch = conf['nn_arch'] self.model_loading = conf['model_loading'] if self.model_loading: with CustomObjectScope({'policy_loss': policy_loss}): self.model = load_model(os.path.join( self.MODEL_PATH)) # Check exception. else: # Design actor. # Input. input_s = Input(shape=(self.nn_arch['state_dim'], ), name='input_s') # Get action. x = input_s for i in range(self.nn_arch['num_layers']): x = Dense(self.nn_arch['dense_layer_dim'], activation='relu', name='dense' + str(i + 1))(x) action = Dense(self.nn_arch['action_dim'], activation='tanh', name='action_value_dense')(x) input_td_error = Input(shape=(1, )) action = Lambda(lambda x: K.log(x))(action) #? action = Lambda(lambda x: -1.0 * x[0] * x[1])( [input_td_error, action]) self.model = Model(inputs=[input_s, input_td_error], outputs=[action]) opt = optimizers.Adam(lr=self.hps['lr'], beta_1=self.hps['beta_1'], beta_2=self.hps['beta_2'], decay=self.hps['decay']) self.model.compile(optimizer='adam', loss=policy_loss) #self.model.summary() self._make_action_model()
def load_model_weight(path): with CustomObjectScope({ 'dice_loss': dice_loss, 'dice_coef': dice_coef, 'bce_dice_loss': bce_dice_loss, 'focal_loss': focal_loss, 'tversky_loss': tversky_loss, 'focal_tversky': focal_tversky }): model = load_model(path) return model
def load_model(model_path, configuration=None): """ Loads saved h5 file with trained model. @param configuration: Optional instance of Configuration with config JSON @param model_path: Path to h5 file @return: Mask2FaceModel """ with CustomObjectScope({ 'ssim_loss': Mask2FaceModel.ssim_loss, 'ssim_l1_loss': Mask2FaceModel.ssim_l1_loss }): model = tf.keras.models.load_model(model_path) return Mask2FaceModel(model, configuration)
def test_save_load(self): encoder_inputs = np.ones(shape=(1, 15)) decoder_inputs = np.ones(shape=(1, 17)) before_saving = self.model.predict([encoder_inputs, decoder_inputs]) self.model.save(filepath=self.file_path) with CustomObjectScope({'Seq2SeqAttentionCNN': Seq2SeqAttentionCNN}): model = load_model(self.file_path) self.assertIsInstance(model, Seq2SeqAttentionCNN) after_saving = self.model.predict([encoder_inputs, decoder_inputs]) self.assertTrue(np.array_equal(before_saving, after_saving))
def load_model(path, compile=True, lr=1e-3): with CustomObjectScope({ 'iou': iou, 'dice_coef': dice_coef, 'dice_loss': dice_loss }): model = tf.keras.models.load_model(path) if compile: metrics = [ dice_coef, iou, MeanIoU(num_classes=2), Recall(), Precision() ] model.compile(loss=dice_loss, optimizer=Adam(lr), metrics=metrics) return model
class Classifier(models.Model): input = models.ImageField() output = models.CharField( max_length=10) # should be at most 7 for "nothing" model_path = "models/v3-ft10-model.json" weights_path = "models/InceptionV3_weights_safe.01-0.17.h5" with CustomObjectScope({'GlorotUniform': glorot_uniform()}): with open(model_path, 'r') as file: model = model_from_json(file.read()) model.load_weights(weights_path) def predict(self): x = image.img_to_array(self.input) x = preprocessing.edge_filter(x) y = self.model.predict([x / 255.0]) y = np.argmax(y[0]) return letter_map[y + 1]
def predict_main(): # parse the command line arguments parser = argparsers.fastpredict_argsparser() args = parser.parse_args() # check if the output directory exists if not os.path.exists(args.output_dir): logging.error("Directory {} does not exist".format(args.output_dir)) return if args.automate_filenames: # create a new directory using current date/time to store the # predictions and logs date_time_str = local_datetime_str(args.time_zone) pred_dir = '{}/{}'.format(args.output_dir, date_time_str) os.mkdir(pred_dir) elif os.path.isdir(args.output_dir): pred_dir = args.output_dir else: logging.error("Directory does not exist {}.".format(args.output_dir)) return # filename to write debug logs logfname = "{}/predict.log".format(pred_dir) # set up the loggers logger.init_logger(logfname) # predict logging.info("Loading {}".format(args.model)) with CustomObjectScope({ 'MultichannelMultinomialNLL': MultichannelMultinomialNLL, 'tf': tf, 'CustomMeanSquaredError': CustomMeanSquaredError, 'AttributionPriorModel': AttributionPriorModel, 'CustomModel': CustomModel }): predict(args, pred_dir)
def test_tf_lite(self): self.skipTest( 'Here is a list of operators for which you will need custom implementations: BatchMatMul.' ) self.model.save(self.file_path) with CustomObjectScope({'Seq2SeqAttentionCNN': Seq2SeqAttentionCNN}): model = load_model(self.file_path) # TF-lite demands to have a fixed size inputs for all the inputs other than the batch dimension fixed_encoder_input = Input(shape=(512, )) fixed_decoder_input = Input(shape=(512, )) fixed_output = model([fixed_encoder_input, fixed_decoder_input]) model = Model([fixed_encoder_input, fixed_decoder_input], fixed_output) converter = tf.lite.TFLiteConverter.from_keras_model(model) tf_lite_model = converter.convert() with open(self.tf_lite_path, 'wb') as f: f.write(tf_lite_model) print(tf_lite_model) print(self.tf_lite_path.stat()) self.assertLessEqual(self.tf_lite_path.stat().st_size, 10000000)
def __init__(self, nb_classes, resnet_layers, input_shape, weights): self.input_shape = input_shape self.num_classes = nb_classes json_path = os.path.join("weights", "keras", weights + ".json") h5_path = os.path.join("weights", "keras", weights + ".h5") if 'pspnet' in weights: if os.path.isfile(json_path) and os.path.isfile(h5_path): print("Keras model & weights found, loading...") with CustomObjectScope({'Interp': layers.Interp}): with open(json_path) as file_handle: self.model = model_from_json(file_handle.read()) self.model.load_weights(h5_path) else: print( "No Keras model & weights found, import from npy weights.") self.model = layers.build_pspnet(nb_classes=nb_classes, resnet_layers=resnet_layers, input_shape=self.input_shape) self.set_npy_weights(weights) else: print('Load pre-trained weights') self.model = load_model(weights)
def classifier(path='./cluster_classifier_64' ): #xception model, pretrained, see `xception.py` with CustomObjectScope({'GlorotUniform': glorot_uniform()}): m = read_model(path) return m
test_mask_paths = glob(os.path.join(test_path, "masks", "*")) test_image_paths.sort() test_mask_paths.sort() ## Create result folder try: os.mkdir(save_path) except: pass ## Model with CustomObjectScope({ 'dice_loss': dice_loss, 'dice_coef': dice_coef, 'miou_loss': miou_loss, 'miou_coef': miou_coef, 'dice_coe': dice_coe, 'dice_hard_coe': dice_hard_coe, 'iou_coe': iou_coe #, 'weighted_hausdorff_distance':weighted_hausdorff_distance }): model = load_model(model_path) ## Added for '<' not supported between instances of 'function' and 'str' lr = 1e-4 optimizer = Nadam(lr) metrics = [ Recall(), Precision(), dice_coef, MeanIoU(num_classes=2), miou_coef,
def __init__(self, conf): """ Parameters ---------- conf: dictionary Semantic segmentation model configuration dictionary. """ # Check exception. assert conf['nn_arch']['output_stride'] == 8 or conf['nn_arch'][ 'output_stride'] == 16 # Initialize. self.conf = conf self.raw_data_path = self.conf['raw_data_path'] self.hps = self.conf['hps'] self.nn_arch = self.conf['nn_arch'] self.model_loading = self.conf['model_loading'] if self.model_loading: opt = optimizers.Adam(lr=self.hps['lr'], beta_1=self.hps['beta_1'], beta_2=self.hps['beta_2'], decay=self.hps['decay']) with CustomObjectScope({ 'CategoricalCrossentropyWithLabelGT': CategoricalCrossentropyWithLabelGT, 'MeanIoUExt': MeanIoUExt }): if self.conf['multi_gpu']: self.model = load_model( os.path.join(self.raw_data_path, self.MODEL_PATH)) self.parallel_model = multi_gpu_model( self.model, gpus=self.conf['num_gpus']) self.parallel_model.compile(optimizer=opt, loss=self.model.losses, metrics=self.model.metrics) else: self.model = load_model( os.path.join(self.raw_data_path, self.MODEL_PATH)) # self.model.compile(optimizer=opt, # , loss=CategoricalCrossentropyWithLabelGT(num_classes=self.nn_arch['num_classes']) # , metrics=[MeanIoUExt(num_classes=NUM_CLASSES)] else: # Design the semantic segmentation model. # Load a base model. if self.conf['base_model'] == BASE_MODEL_MOBILENETV2: # Load mobilenetv2 as the base model. mv2 = MobileNetV2( include_top=False ) # , depth_multiplier=self.nn_arch['mv2_depth_multiplier']) if self.nn_arch['output_stride'] == 8: self.base = Model( inputs=mv2.inputs, outputs=mv2.get_layer('block_5_add').output ) # Layer satisfying output stride of 8. else: self.base = Model( inputs=mv2.inputs, outputs=mv2.get_layer('block_12_add').output ) # Layer satisfying output stride of 16. self.base.trainable = True for layer in self.base.layers: layer.trainable = True # ? self.base._init_set_name('base') elif self.conf['base_model'] == BASE_MODEL_XCEPTION: # Load xception as the base model. mv2 = Xception( include_top=False ) # , depth_multiplier=self.nn_arch['mv2_depth_multiplier']) if self.nn_arch['output_stride'] == 8: self.base = Model( inputs=mv2.inputs, outputs=mv2.get_layer('block4_sepconv2_bn').output ) # Layer satisfying output stride of 8. else: self.base = Model( inputs=mv2.inputs, outputs=mv2.get_layer('block13_sepconv2_bn').output ) # Layer satisfying output stride of 16. self.base.trainable = True for layer in self.base.layers: layer.trainable = True # ? self.base._init_set_name('base') # Make the encoder-decoder model. self._make_encoder() self._make_decoder() inputs = self.encoder.inputs features = self.encoder(inputs) outputs = self.decoder([inputs[0], features]) if self.nn_arch['boundary_refinement'] \ else self.decoder(features) self.model = Model(inputs, outputs) # Compile. opt = optimizers.Adam(lr=self.hps['lr'], beta_1=self.hps['beta_1'], beta_2=self.hps['beta_2'], decay=self.hps['decay']) self.model.compile(optimizer=opt, loss=CategoricalCrossentropyWithLabelGT( num_classes=self.nn_arch['num_classes']), metrics=[MeanIoUExt(num_classes=NUM_CLASSES)]) self.model._init_set_name('deeplabv3plus_mnv2') if self.conf['multi_gpu']: self.parallel_model = multi_gpu_model( self.model, gpus=self.conf['num_gpus']) self.parallel_model.compile(optimizer=opt, loss=self.model.losses, metrics=self.model.metrics)
mask = np.transpose(mask, (1,2,0)) return mask if __name__ == "__main__": (train_x, train_y), (valid_x, valid_y), (test_x, test_y) = load_data("dataset/") print(len(train_x), len(test_x)) batch = 8 test_dataset = tf_dataset(test_x, test_y, batch = batch) test_steps = len(test_x)//batch if len(test_x) % batch != 0: test_steps +=1 with CustomObjectScope({'iou':iou}): model = tf.keras.models.load_model("files/model.h5") model.evaluate(test_dataset, steps = test_steps) for i , (x,y) in tqdm(enumerate(zip(test_x, test_y)), total = len(test_x)): x = read_image(x) y = read_mask(y) y_pred = model.predict(np.expand_dims(x, axis = 0)) y_pred = y_pred[0] > 0.5 h,w,_ = x.shape white_line = np.ones((h, 10, 3)) * 255.0 all_images = [ x * 255.0, white_line,
ap.add_argument("-db", "--database", default="data/NN_smats.db", help="sqlite database containing the adresses") ap.add_argument("-S", "--smats", default="data/smats", help="directory containing the smats for interpolation") ap.add_argument("-i", "--index", default=0, type=int) ap.add_argument("-I", "--interpolate", action="store_false", default=True) args = vars(ap.parse_args()) #%% print("[INFO] loading network...") #the scope is nessecary beacuse I used a custom loss for training with CustomObjectScope({'loss': mean_squared_error}): model = load_model(args["model"]) print("[INFO] loading input spectrum...") lb = LabelBinarizer() target_spectrum = np.load(args["spectrum"])[args['index']] with sqlite3.connect(database=args['database']) as conn: crawler = Crawler(directory=args['smats'], cursor=conn.cursor()) #Phase 1: use the model to get an initial guess print("[INFO] classifying spectrum...") p1, p2, p_stack = classify(model, target_spectrum, lb) print(p1) print(p2) print(p_stack)
model_path = "files/resunetplusplus.h5" ## Parameters image_size = 256 batch_size = 32 lr = 1e-4 epochs = 100 ## Validation valid_path = "cs_data/CVC-12k" valid_image_paths = sorted(glob(os.path.join(valid_path, "image", "*.jpg"))) valid_mask_paths = sorted(glob(os.path.join(valid_path, "mask", "*.jpg"))) with CustomObjectScope({ 'dice_loss': dice_loss, 'dice_coef': dice_coef, 'bce_dice_loss': bce_dice_loss, 'focal_loss': focal_loss, 'tversky_loss': tversky_loss, 'focal_tversky': focal_tversky }): model = load_model(model_path) evaluate_normal(model, valid_image_paths, valid_mask_paths) evaluate_crf(model, valid_image_paths, valid_mask_paths) evaluate_tta(model, valid_image_paths, valid_mask_paths) evaluate_crf_tta(model, valid_image_paths, valid_mask_paths)
def task_capture(iv_interface, iv_as_proxy=False, iv_dns_up_ip=None, iv_port=None): global gv_model global gv_model_dga global gv_valid_chars global gv_family_dict global gv_maxlen global gv_logger global gv_interface_ip global gv_session global gv_graph global gv_pre_domain global gv_id_dataset global gv_id_model_dga global gv_id_model_family current_task.update_state( state='PROGRESS', meta={'step': 'loading dga prediction model from disk...'}) with CustomObjectScope({'GlorotUniform': glorot_uniform()}): gv_model = load_model('get_model/input_data/dga_prediction_model.h5') gv_id_model_dga = PreparedModel.objects.filter( model_type='binary').latest('id') current_task.update_state( state='PROGRESS', meta={'step': 'loading family prediction model from disk...'}) with CustomObjectScope({'GlorotUniform': glorot_uniform()}): gv_model_dga = load_model( 'get_model/input_data/family_prediction_model.h5') gv_id_model_family = PreparedModel.objects.filter( model_type='multiclass').latest('id') current_task.update_state(state='PROGRESS', meta={'step': 'loading data for models...'}) with open('get_model/input_data/training_data.pkl', 'rb') as f: lv_training_data = pickle.load(f) gv_id_dataset = PreparedDataset.objects.latest('id') lv_all_data_dict = pd.concat( [lv_training_data['legit'][:100000], lv_training_data['dga'][:100000]], ignore_index=False, sort=True) gv_family_dict = { idx: x for idx, x in enumerate(lv_training_data['dga']['family'].unique()) } lv_x = np.array(lv_all_data_dict['domain'].tolist()) gv_valid_chars = {x: idx + 1 for idx, x in enumerate(set(''.join(lv_x)))} gv_maxlen = np.max([len(x) for x in lv_x]) current_task.update_state(state='PROGRESS', meta={'step': 'warming-up models...'}) gv_model.predict(np.array([np.zeros(gv_maxlen, dtype=int)])) gv_model_dga.predict(np.array([np.zeros(gv_maxlen, dtype=int)])) gv_session = K.get_session() gv_graph = tf.get_default_graph() gv_graph.finalize() gv_logger = logger_setup() gv_logger.info("Запросы содержащие DGA-домены:") # Получение ip адреса интерфейса. lv_addr = ni.ifaddresses(iv_interface) gv_interface_ip = lv_addr[ni.AF_INET][0]['addr'] current_task.update_state(state='PROGRESS', meta={'step': 'capturing requests...'}) # Основной процесс, сканирование сети. if iv_as_proxy == True: lv_host = '' # Настройка udp сервера для получения dns запросов. lv_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) lv_sock.bind((lv_host, iv_port)) while True: lv_data, lv_addr = lv_sock.recvfrom(1024) lv_th = Thread(target=handler, args=(lv_data, lv_addr, lv_sock, iv_dns_up_ip, iv_interface)) lv_th.start() # Put in view to safety close socket thread # lv_sock.close() else: sniff(iface=iv_interface, filter="port 53", store=0, prn=packet_callback) return {'step': 'success'}
return mask # Set path to test dataset # TEST_DATASET_PATH = "./test_images" TEST_DATASET_PATH = "new_data/CVC-ClinicDB/test/images" GROUND_TRUTH_PATH = "new_data/CVC-ClinicDB/test/masks/" # MASK_PATH = "./mask" PREDICTED_MASK_PATH = "new_data/CVC-ClinicDB/test/predicted_masks" Results = "new_data/bagging_unet_results" model_path = "files_before_tpu/miou.h5" # Load Keras model with CustomObjectScope({ 'dice_loss': dice_loss, 'dice_coef': dice_coef, 'miou_loss': miou_loss, 'miou_coef': miou_coef }): unet_cvc28 = tf.keras.models.load_model( "files/models/bagged_unet_cvc/unet_cvc28.h5") with CustomObjectScope({ 'dice_loss': dice_loss, 'dice_coef': dice_coef, 'miou_loss': miou_loss, 'miou_coef': miou_coef }): unet_cvc27 = tf.keras.models.load_model( "files/models/bagged_unet_cvc/unet_cvc27.h5") with CustomObjectScope({
format(auc, targetfpr, recall, abs(threshold))) # show the plot plt.show() #evaluate on untrained feature extractor base network def evalute_model(network, dataset_test, x_test_ori, y_test_ori): probs, yprob = compute_probs(network, x_test_ori[:500, :, :, :], y_test_ori[:500]) probs, yprob = compute_probs(network, x_test_ori[:500, :, :, :], y_test_ori[:500]) fpr, tpr, thresholds, auc = compute_metrics(probs, yprob) draw_roc(fpr, tpr, auc, thresholds) draw_interdist(dataset_test, network, n_iteration=0) if __name__ == '__main__': dataset_train, dataset_test, x_train_ori, y_train_ori, x_test_ori, y_test_ori = getDataset( ) #base=build_basemodel(input_shape, embeddingsize) # evalute_model(base, dataset_test, x_test_ori,y_test_ori) with CustomObjectScope({'TripletLossLayer': TripletLossLayer}): trained_model = load_model('../working/mnist_triplet.h5') trained_model.summary() base_model = load_model('../working/mnist_base.h5') base_model.summary() evalute_model(base_model, dataset_test, x_test_ori, y_test_ori)
os.path.join(DIR_MODELS, 'LeftRight/NasnetMobile-007-0.991.hdf5'), 'input_shape': (224, 224, 3), 'model_weight': 1, } dicts_models.append(dict1) #endregion #load models for dict1 in dicts_models: model_file = dict1['model_file'] print('%s load start!' % (model_file)) # ValueError: Unknown activation function:relu6 MobileNet V2 with CustomObjectScope({ 'relu6': keras.layers.ReLU(6.), 'DepthwiseConv2D': keras.layers.DepthwiseConv2D }): dict1['model'] = keras.models.load_model(model_file, compile=False) if 'input_shape' not in dict1: if len(dict1['model'].input_shape ) == 4: #(batch, height, width, channel) dict1['input_shape'] = dict1['model'].input_shape[1:] else: dict1['input_shape'] = (299, 299, 3) print('%s load complte!' % (model_file)) #region test mode if my_config.debug_mode: img_source = '/tmp1/66b17a1e-a74d-11e8-94f6-6045cb817f5b.jpg'
def final(page,start_date,end_date): import pandas as pd from datetime import datetime,timedelta import numpy as np from tensorflow.keras.initializers import glorot_uniform from tensorflow.keras.optimizers import Adam from tensorflow.keras import backend as K from tensorflow.keras.models import load_model from tensorflow.keras.utils import CustomObjectScope import re start_date=datetime.strptime(start_date, '%Y-%m-%d') end_date=datetime.strptime(end_date, '%Y-%m-%d') delta = end_date-start_date days= delta.days+1 new_date = end_date + timedelta(10) datelist = pd.date_range(start_date,new_date-timedelta(days=1),freq='d') weekday_test=[] print("Creating the features ... ") for i in datelist: weekday_test.append(i.weekday()) weekday_test=pd.Series(weekday_test) month_test=[] for i in datelist: month_test.append(i.month) month_test=pd.Series(month_test) month_start_test=[] month_start_test=pd.Series(datelist).dt.is_month_start month_end_test=[] month_end_test=pd.Series(datelist).dt.is_month_end quarter_start_test=[] quarter_start_test=pd.Series(datelist).dt.is_quarter_start quarter_end_test=[] quarter_end_test=pd.Series(datelist).dt.is_quarter_end week_test=[] week_test=pd.Series(datelist).dt.week quarter_test=[] quarter_test=pd.Series(datelist).dt.quarter days_in_month_test=[] days_in_month_test =pd.Series(datelist).dt.days_in_month year_test=[] year_test=pd.Series(datelist).dt.year is_sunday_or_monday_test=[] for i in weekday_test: if i == 0 or i == 6: is_sunday_or_monday_test.append(1) else: is_sunday_or_monday_test.append(0) is_sunday_or_monday_test=pd.Series(is_sunday_or_monday_test) is_august_test=[] for i in month_test: if i == 8: is_august_test.append(1) else: is_august_test.append(0) is_august_test=pd.Series(is_august_test) year_half_test=[] for i in month_test: if i in [1,2,3,4,5,6] : year_half_test.append(1) else : year_half_test.append(2) year_half_test=pd.Series(year_half_test) ### The above features are irrespective of the page , I will call them global features global_feat=pd.DataFrame() global_feat=pd.concat([weekday_test,is_sunday_or_monday_test,month_test,is_august_test,year_half_test,quarter_test,quarter_start_test,quarter_end_test,month_start_test,month_end_test,days_in_month_test,week_test],axis=1) global_feat.columns=['weekday','is_sunday_or_monday','month','is_august','year_half','quarter','quarter_start','quarter_end','month_start','month_end','days_in_month','week'] def access(page): all_access=re.search('all-access',page) desktop=re.search('desktop',page) mobile=re.search('mobile-web',page) if(all_access): return (0) elif(desktop): return(1) else: return(2) def agent(page): index=re.search('spider',page) if(index): return (1) else: return(0) access_index=access(page) agent_index=agent(page) pageview = np.load('viewperc.npy',allow_pickle='TRUE').item() viewperc= pageview[page] viewmid=pd.read_csv('viewmid.csv') view1=viewmid.loc[0].values[0] view2=viewmid.loc[1].values[0] view3=viewmid.loc[2].values[0] view4=viewmid.loc[3].values[0] if agent_index == 1: spider=[1]*(global_feat.shape[0]) non_spider=[0]*(global_feat.shape[0]) else: spider=[0]*(global_feat.shape[0]) non_spider=[1]*(global_feat.shape[0]) spider=pd.Series(spider) non_spider=pd.Series(non_spider) page_specific_feat=pd.DataFrame() page_specific_feat=pd.concat([spider,non_spider],axis=1) page_specific_feat.columns=['spider','non_spider'] if access_index==0: page_specific_feat['All_Access']=1 page_specific_feat['Desktop']=0 page_specific_feat['Mobile']=0 elif access_index==1: page_specific_feat['All_Access']=0 page_specific_feat['Desktop']=1 page_specific_feat['Mobile']=0 else: page_specific_feat['All_Access']=0 page_specific_feat['Desktop']=0 page_specific_feat['Mobile']=1 total_feat=pd.concat([global_feat,page_specific_feat],axis=1) print("Feature Created ...") print("Preprocessing the data ... ") from sklearn.preprocessing import LabelEncoder le1=LabelEncoder() total_feat['month_start']=le1.fit_transform(total_feat['month_start']) le2=LabelEncoder() total_feat['month_end']=le2.fit_transform(total_feat['month_end']) le3=LabelEncoder() total_feat['quarter_start']=le3.fit_transform(total_feat['quarter_start']) le4=LabelEncoder() total_feat['quarter_end']=le4.fit_transform(total_feat['quarter_end']) def create_test_dataset(X,timestep=1): Xs=[] for i in range(len(X)): end_ix=i+timestep if end_ix > X.shape[0]: break v=X[i:end_ix] Xs.append(v) return np.array(Xs) total_feat=np.log1p(total_feat) test_x=create_test_dataset(total_feat.values,7) print("Preprocessing Completed ... ") def customLoss(y_true, y_pred): epsilon = 0.1 summ = K.maximum(K.abs(y_true) + K.abs(y_pred) + epsilon, 0.5 + epsilon) smape = K.abs(y_pred - y_true) / summ * 2.0 return smape opt=Adam(learning_rate=0.001) with CustomObjectScope({'GlorotUniform': glorot_uniform()}): model1=load_model('model1_new.hdf5',compile=False) model1.compile(loss=customLoss,optimizer=opt) model2=load_model('model2_new.hdf5',compile=False) model2.compile(loss=customLoss,optimizer=opt) model3=load_model('model3_new.hdf5',compile=False) model3.compile(loss=customLoss,optimizer=opt) model4=load_model('model4_new.hdf5',compile=False) model4.compile(loss=customLoss,optimizer=opt) model5=load_model('model5_new.hdf5',compile=False) model5.compile(loss=customLoss,optimizer=opt) model6=load_model('model6_new.hdf5',compile=False) model6.compile(loss=customLoss,optimizer=opt) model7=load_model('model7_new.hdf5',compile=False) model7.compile(loss=customLoss,optimizer=opt) model8=load_model('model8_new.hdf5',compile=False) model8.compile(loss=customLoss,optimizer=opt) print("Predicting the pagehits ... ") if access_index==0 and agent_index==0: if viewperc>=view1: y_pred_lstm=model1.predict(test_x) else: y_pred_lstm=model5.predict(test_x) elif access_index==1 and agent_index==0: if viewperc>=view2: y_pred_lstm=model2.predict(test_x) else: y_pred_lstm=model6.predict(test_x) elif access_index==2 and agent_index==0: if viewperc>=view3: y_pred_lstm=model3.predict(test_x) else: y_pred_lstm=model7.predict(test_x) elif access_index==0 and agent_index==1: if viewperc>=view4: y_pred_lstm=model4.predict(test_x) else: y_pred_lstm=model8.predict(test_x) y_pred_lstm=y_pred_lstm[0:days] y_pred_lstm=np.exp(y_pred_lstm)-1 y_pred_lstm=pd.DataFrame(y_pred_lstm) y_pred_lstm.index=datelist[0:days] y_pred_lstm=y_pred_lstm.transpose() y_pred_lstm.index=[page] print("Task Completed ... ") return y_pred_lstm
from matplotlib import pyplot as plt from tensorflow.keras.applications.mobilenet_v2 import preprocess_input from tensorflow.keras.models import load_model from tensorflow.keras.utils import CustomObjectScope from tensorflow.keras import backend as K def relu6(x): return K.relu(x, max_value=6) @tf.function def preprocess_image(image): image = tf.image.decode_png(image, channels=3) image = tf.image.resize(image, (320, 240,), method=tf.image.ResizeMethod.LANCZOS3) return preprocess_input(image) with CustomObjectScope({'relu6': relu6,'DepthwiseConv2D': tf.keras.layers.DepthwiseConv2D}): model = load_model(os.path.join(os.getcwd(), 'result', 'cloudsegnet.hdf5'), custom_objects={"tf": tf}) image = tf.io.read_file(sys.argv[1]) image = preprocess_image(image) pred = model.predict(np.expand_dims(image, 0)) labels = np.argmax(pred.squeeze(), -1) # remove padding and resize back to original image labels = np.array(Image.fromarray(labels.astype('uint8')).resize((240, 320,))) plt.imshow(labels) plt.waitforbuttonpress()
def target_train_tpu_main(gen_targets_dir, model_file_path, early_stopping_patience=None, length=None, batch_size=1, period=1, retrain_file=None, retrain_do_compile=False, base_model_file_path='target_common.h5', optimizer=Adam(), optimizer_lr=0.001, epochs=100000): gc.collect() with CustomObjectScope({'RandomLayer': RandomLayer}): if retrain_file is None: gen = VoiceGeneratorTargetTpu(gen_targets_dir, 0.1, batch_size, length, train=True) shape0 = gen[0][0].shape[1] val_gen = VoiceGeneratorTargetTpu(gen_targets_dir, 0.1, train=False, max_size=shape0) model = load_model(base_model_file_path) config = model.get_config() config['layers'][0]['config']['batch_input_shape'] = (None, shape0, 139) config['layers'][3]['config']['rate'] = 0.1 config['layers'][6]['config']['target_shape'] = (shape0 * 2, 64) config['layers'][8]['config']['rate'] = 0.1 config['layers'][11]['config']['target_shape'] = (shape0 * 4, 32) config['layers'][13]['config']['rate'] = 0.1 config['layers'][16]['config']['target_shape'] = (shape0 * 8, 16) config['layers'][18]['config']['rate'] = 0.1 model = Sequential.from_config(config) model.load_weights(base_model_file_path, by_name=True) model.summary() model.compile(loss='mse', optimizer=optimizer) baseline = None else: model = load_model(retrain_file) if retrain_do_compile: model.compile(loss='mse', optimizer=optimizer) config = model.get_config() shape0 = config['layers'][0]['config']['batch_input_shape'][1] gen = VoiceGeneratorTargetTpu(gen_targets_dir, 0.1, batch_size, length, train=True, max_size=shape0) val_gen = VoiceGeneratorTargetTpu(gen_targets_dir, 0.1, train=False, max_size=shape0) baseline = model.test_on_batch(val_gen[0][0], val_gen[0][1]) tpu_grpc_url = 'grpc://' + os.environ['COLAB_TPU_ADDR'] tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( tpu_grpc_url) strategy = keras_support.TPUDistributionStrategy(tpu_cluster_resolver) model = tf.contrib.tpu.keras_to_tpu_model(model, strategy=strategy) cp = ModelCheckpoint(filepath=model_file_path, monitor='val_loss', save_best_only=True, period=period) if baseline is not None: cp.best = baseline def lr_scheduler(epoch): return optimizer_lr scheduler = LearningRateScheduler(lr_scheduler) if early_stopping_patience is not None: es = EarlyStopping(monitor='val_loss', patience=early_stopping_patience, verbose=0, mode='auto', baseline=baseline) callbacks = [es, cp, scheduler] else: callbacks = [cp, scheduler] model.fit_generator(gen, shuffle=True, epochs=epochs, verbose=1, callbacks=callbacks, validation_data=val_gen) K.clear_session()
# Specify Image Dimensions IMG_WIDTH = 256 IMG_HEIGHT = 256 IMG_CHANNELS = 3 warnings.filterwarnings('ignore', category=UserWarning, module='skimage') seed = 42 random.seed = seed Recall = keras.metrics.Recall() counter = 0 # Load the trained model, make sure to set up the path correctly # some models require to load the weights seperately, keep that in mind if errors occur with CustomObjectScope({ 'iou': iou, 'dice_loss': sm.losses.DiceLoss(), 'f1-score': sm.metrics.FScore(), 'recall': keras.metrics.Recall() }): model = tf.keras.models.load_model('FPN_inceptionv3.h5') # Open window to visualize the segmentation cv2.namedWindow("segmentations") cv2.namedWindow("normal") # Define video path of predicted video, you can choose 0 which opens the computer camera (maybe another integer on other systems) # It is also possible to read in a USB camera through the choice of an integer vc = cv2.VideoCapture("test_placenta_msrl_3.mp4") # Try to get first frame if vc.isOpened(): rval, frame = vc.read()
image_size = 256 batch_size = 1 test_image_paths = glob(os.path.join(test_path, "images", "*")) test_mask_paths = glob(os.path.join(test_path, "masks", "*")) test_image_paths.sort() test_mask_paths.sort() ## Create result folder try: os.mkdir(save_path) except: pass ## Model with CustomObjectScope({'dice_loss': dice_loss, 'dice_coef': dice_coef}): model = load_model(model_path) ## Test print("Test Result: ") test_steps = len(test_image_paths) // batch_size test_gen = DataGen(image_size, test_image_paths, test_mask_paths, batch_size=batch_size) model.evaluate_generator(test_gen, steps=test_steps, verbose=1) ## Generating the result for i, path in tqdm(enumerate(test_image_paths), total=len(test_image_paths)): image = parse_image(test_image_paths[i], image_size)
def target_train_tpu_convert_main(input_file, output_weight_file, output_json_file): with CustomObjectScope({'RandomLayer': RandomLayer}): input_model = load_model(input_file) input_model.save_weights(output_weight_file) config = input_model.get_config() config['layers'][0]['config']['batch_input_shape'] = (None, None, 139) config['layers'][0]['config']['layer']['config'][ 'kernel_initializer'] = 'zeros' config['layers'][1]['config']['kernel_initializer'] = 'zeros' config['layers'][4]['config']['layer']['config'][ 'kernel_initializer'] = 'zeros' config['layers'][5]['config']['kernel_initializer'] = 'zeros' config['layers'][6]['config']['target_shape'] = (-1, 64) config['layers'][9]['config']['layer']['config'][ 'kernel_initializer'] = 'zeros' config['layers'][10]['config']['kernel_initializer'] = 'zeros' config['layers'][11]['config']['target_shape'] = (-1, 32) config['layers'][14]['config']['layer']['config'][ 'kernel_initializer'] = 'zeros' config['layers'][15]['config']['kernel_initializer'] = 'zeros' config['layers'][16]['config']['target_shape'] = (-1, 16) config['layers'][19]['config']['layer']['config'][ 'kernel_initializer'] = 'zeros' config['layers'][20]['config']['kernel_initializer'] = 'zeros' model = Sequential.from_config(config) json_dic = json.loads(model.to_json()) del json_dic['config']['layers'][0]['config']['layer']['config'][ 'time_major'] del json_dic['config']['layers'][0]['config']['layer']['config'][ 'zero_output_for_mask'] json_dic['config']['layers'][2]['class_name'] = 'BatchNormalization' json_dic['config']['layers'][2]['config']['axis'] = 2 del json_dic['config']['layers'][4]['config']['layer']['config'][ 'time_major'] del json_dic['config']['layers'][4]['config']['layer']['config'][ 'zero_output_for_mask'] json_dic['config']['layers'][7]['class_name'] = 'BatchNormalization' json_dic['config']['layers'][7]['config']['axis'] = 2 del json_dic['config']['layers'][9]['config']['layer']['config'][ 'time_major'] del json_dic['config']['layers'][9]['config']['layer']['config'][ 'zero_output_for_mask'] json_dic['config']['layers'][12]['class_name'] = 'BatchNormalization' json_dic['config']['layers'][12]['config']['axis'] = 2 del json_dic['config']['layers'][14]['config']['layer']['config'][ 'time_major'] del json_dic['config']['layers'][14]['config']['layer']['config'][ 'zero_output_for_mask'] json_dic['config']['layers'][17]['class_name'] = 'BatchNormalization' json_dic['config']['layers'][17]['config']['axis'] = 2 del json_dic['config']['layers'][19]['config']['layer']['config'][ 'time_major'] del json_dic['config']['layers'][19]['config']['layer']['config'][ 'zero_output_for_mask'] with open(output_json_file, 'w') as f: json.dump(json_dic, f)
from get_data import * import tensorflow as tf import tensorflow.keras as keras from tensorflow.python.keras.models import load_model from tensorflow.python.keras.backend import set_session from tensorflow.keras.utils import CustomObjectScope from tensorflow.keras.initializers import glorot_uniform from google.colab import drive drive.mount('/content/gdrive') sess = tf.Session() graph = tf.get_default_graph() set_session(sess) """ this is just to load the models, it will not work if you try to run this""" with CustomObjectScope({'GlorotUniform': glorot_uniform()}): fovmodel = tf.keras.models.load_model( '/content/gdrive/My Drive/my_model.h5', ) nofovmodel = tf.keras.models.load_model( '/content/gdrive/My Drive/my_model_nofov.h5', ) model_input = tf.keras.layers.Input(shape=(64, 64, 3)) model_output = nofovmodel( model_input ) ###This line indicates that the perturbations are based on the regular model model = tf.keras.models.Model(inputs=model_input, outputs=model_output) model.compile(optimizer=tf.keras.optimizers.SGD(), loss='categorical_crossentropy', metrics=['accuracy']) fmodel = KerasModel2(model, bounds=(0.0, 255.0))
def shap_scores_main(): # disable eager execution so shap deep explainer wont break tf.compat.v1.disable_eager_execution() # parse the command line arguments parser = shap_scores_argsparser() args = parser.parse_args() # check if the output directory exists if not os.path.exists(args.output_directory): raise NoTracebackException("Directory {} does not exist".format( args.output_directory)) # check if the output directory is a directory path if not os.path.isdir(args.output_directory): raise NoTracebackException("{} is not a directory".format( args.output_directory)) # check if the reference genome file exists if not os.path.exists(args.reference_genome): raise NoTracebackException("File {} does not exist".format( args.reference_genome)) # check if the model file exists if not os.path.exists(args.model): raise NoTracebackException("File {} does not exist".format(args.model)) # check if the bed file exists if not os.path.exists(args.bed_file): raise NoTracebackException("File {} does not exist".format( args.bed_file)) # if controls are specified check if the control_info json exists if args.input_data is not None: if not os.path.exists(args.input_data): raise NoTracebackException( "Input data file {} does not exist".format(args.control_info)) # check if both args.chroms and args.sample are specified, only # one of the two is allowed if args.chroms is not None and args.sample is not None: raise NoTracebackException( "Only one of [--chroms, --sample] is allowed") if args.automate_filenames: # create a new directory using current date/time to store the # shapation scores date_time_str = local_datetime_str(args.time_zone) shap_scores_dir = '{}/{}'.format(args.output_directory, date_time_str) os.mkdir(shap_scores_dir) else: shap_scores_dir = args.output_directory # filename to write debug logs logfname = "{}/shap_scores.log".format(shap_scores_dir) # set up the loggers init_logger(logfname) # shap logging.info("Loading {}".format(args.model)) with CustomObjectScope({ 'MultichannelMultinomialNLL': MultichannelMultinomialNLL, 'tf': tf, 'CustomMeanSquaredError': CustomMeanSquaredError, 'AttributionPriorModel': AttributionPriorModel, 'CustomModel': CustomModel }): shap_scores(args, shap_scores_dir)