def load_model(model_path):
model_config_path = os.path.join(os.path.dirname(model_path), "config.yml")
model_name = load(model_config_path, "model_name")
if hasattr(MODELS[model_name], "custom_objects"):
return keras_load_model(
model_path,
custom_objects=MODELS[model_name].custom_objects,
compile=False)
else:
return keras_load_model(model_path, compile=False)
def load_model(self, load_model_name: str, detection: str, num_set: str):
model_path = os.path.join("Models", detection,
load_model_name + "_" + num_set + "_last.h5")
model = keras_load_model(model_path)
return model
def _load_model(self, load_dir, verbose=True):
"""
Loads models from a directory.
:param load_dir: path to directory to load
:param verbose: print details
:return: None
"""
# load models and associated history
# Usually called manually after a pickle load of all other attributes
self.discriminator = keras_load_model(f"{load_dir}/discriminator")
self.discriminator.trainable = False
self.generator = keras_load_model(f"{load_dir}/generator")
self.combined = keras_load_model(f"{load_dir}/combined")
with open(f"{load_dir}/history.p", "rb") as file:
self.history = pickle.load(file)
if verbose:
self.print_summary()
def __init__(self, teacher_model):
self.train_model, self.born_again_model = None, None
self.temperature = args.temperature
self.teacher_model = keras_load_model(teacher_model)
for i in range(len(self.teacher_model.layers)):
self.teacher_model.layers[i].trainable = False
self.teacher_model.compile(optimizer="adam",
loss="categorical_crossentropy")
self.train_model, self.born_again_model = self.prepare()
self.train_model = convert_gpu_model(self.train_model)
def load_model(self, load_model_name: str, detection: str, num_set: str):
model_path = os.path.join("Models", detection,
load_model_name + "_" + num_set + "_last.h5")
model = keras_load_model(model_path,
custom_objects={
"DiluteLayer": DiluteLayer,
"CustomF1": CustomF1
},
compile=False)
return model
def load_model(path=None):
if path is None:
return model.single_frame_model()
else:
if os.path.exists(path):
return keras_load_model(path,
custom_objects={
'temporal_crossentropy':
train.temporal_crossentropy,
'temporal_accuracy':
train.temporal_accuracy
})
else:
raise FileNotFoundError(f"Model path {path} doesn't exist")
def load_model(filepath, custom_objects=None, compile=True) -> Model:
"""
Equivalent to load_model from keras, but custom_objects are already known
Args:
filepath: One of the following: - String, path to the saved model - `h5py.File`
object from which to load the model.
custom_objects: Optional dictionary mapping names (strings) to custom classes
or functions to be considered during deserialization.
compile: Boolean, whether to compile the model after loading.
Returns:
A Keras model instance. If an optimizer was found
"""
deel_custom_objects = CUSTOM_OBJECTS.copy()
if custom_objects is not None:
deel_custom_objects.update(custom_objects)
return keras_load_model(filepath, deel_custom_objects, compile)
def load_model(model_file, network_config):
model = keras_load_model(model_file)
# If necessary, wrap the loaded model to transpose the axes for both
# inputs and outputs.
if network_config.transpose:
inputs = []
perms = []
for old_input in model.input_layers:
input_shape = np.asarray(old_input.input_shape)[[3, 2, 1, 4]]
new_input = Input(shape=tuple(input_shape),
dtype=old_input.input_dtype,
name=old_input.name)
perm = Permute((3, 2, 1, 4),
input_shape=tuple(input_shape))(new_input)
inputs.append(new_input)
perms.append(perm)
old_outputs = model(perms)
if not isinstance(old_outputs, list):
old_outputs = [old_outputs]
outputs = []
for old_output in old_outputs:
new_output = Permute((3, 2, 1, 4))(old_output)
outputs.append(new_output)
new_model = Model(input=inputs, output=outputs)
# Monkeypatch the save to save just the underlying model.
func_type = type(model.save)
old_model = model
def new_save(_, *args, **kwargs):
old_model.save(*args, **kwargs)
new_model.save = func_type(new_save, new_model)
model = new_model
return model
def load_keras_model(self, identifier):
if not identifier.endswith('h5'):
raise ValueError(
"To load keras model, model extension must end with `h5`")
try:
model_path = os.path.join(self.output_folder, identifier)
self._file_exists(model_path)
if not is_tf_support:
raise RuntimeError(
"When try to use `TensorFlow` backend to load model, couldn't load module: `tensorflow`, please check!"
)
model = keras_load_model(model_path)
return model
except IOError as e:
logger.error("When to load keras model, get error: {}".format(e))
raise IOError("When to load keras model, get error: {}".format(e))
def load_model(path='../models/inception_v3'):
"""Retrieves the trained model"""
model = keras_load_model(path)
return model
def load_model(filename, **kwargs):
"""
Load model from file
Parameters
----------
filename : str
path to the h5 file
Returns
-------
deoxys.model.Model
The loaded model
"""
# Keras got the error of loading custom object
try:
loaded_model = keras_load_model(filename,
custom_objects={
**Layers().layers,
**Activations().activations,
**Losses().losses,
**Metrics().metrics
})
# keyword arguments to create the model
model_kwargs = {}
with h5py.File(filename, 'r') as hf:
# get the data_reader
if 'deoxys_config' in hf.attrs.keys():
config = hf.attrs['deoxys_config']
config = load_json_config(config)
if 'dataset_params' in config:
model_kwargs['data_reader'] = load_data(
config['dataset_params'])
model_kwargs['config'] = config
# take the sample data
if 'deoxys' in hf.keys():
if 'batch_x' in hf['deoxys'] and 'batch_y' in hf['deoxys']:
model_kwargs['sample_data'] = (hf['deoxys']['batch_x'][:],
hf['deoxys']['batch_y'][:])
elif 'batch_y' in hf['deoxys']:
batch_y = hf['deoxys']['batch_y'][:]
batch_x = []
for key in hf['deoxys'].keys:
if 'batch_x' in key:
i = int(key[-1])
assert i == len(batch_x)
batch_x.append(hf['deoxys'][f'batch_x_{i}'])
model_kwargs['sample_data'] = (batch_x, batch_y)
# User input will overwrites all existing args
model_kwargs.update(kwargs)
model = Model(loaded_model, pre_compiled=True, **model_kwargs)
except Exception:
sample_data = None
with h5py.File(filename, 'r') as hf:
if 'deoxys_config' in hf.attrs.keys():
config = hf.attrs['deoxys_config']
if 'deoxys' in hf.keys():
if 'batch_x' in hf['deoxys'] and 'batch_y' in hf['deoxys']:
sample_data = (hf['deoxys']['batch_x'][:],
hf['deoxys']['batch_y'][:])
model = model_from_full_config(config,
weights_file=filename,
sample_data=sample_data)
return model
def launch_model():
full_text = request.form['full_text']
id_ = request.form['id']
model_type = request.form['model_type']
global BERT, JOINT, GRANU, MGN, NUM_TASK, MASKING, HIER
BERT = model_type == BERT_PATH
JOINT = model_type == JOINT_BERT_PATH
GRANU = model_type == GRANU_BERT_PATH
MGN = model_type == MGN_SIGM_BERT_PATH
# either of the four variants:
# BERT = False
# JOINT = False
# GRANU = False
# MGN = True
assert BERT or JOINT or GRANU or MGN
assert not (BERT and JOINT) and not (BERT and GRANU) and not (BERT and MGN) \
and not (JOINT and GRANU) and not (JOINT and MGN) and not (GRANU and MGN)
# either of the two variants
SIGMOID_ACTIVATION = True
RELU_ACTIVATION = False
assert not (SIGMOID_ACTIVATION and RELU_ACTIVATION) and (
SIGMOID_ACTIVATION or RELU_ACTIVATION)
if BERT:
NUM_TASK = 1
MASKING = 0
HIER = 0
elif JOINT:
NUM_TASK = 2
MASKING = 0
HIER = 0
elif GRANU:
NUM_TASK = 2
MASKING = 0
HIER = 1
elif MGN:
NUM_TASK = 2
MASKING = 1
HIER = 0
else:
raise ValueError(
"You should choose one of bert, joint, granu and mgn in options")
dct = {
'NUM_TASK': NUM_TASK,
'MASKING': MASKING,
'SIGMOID_ACTIVATION': SIGMOID_ACTIVATION,
'HIER': HIER
}
model = load_model(model_type, **dct)
if not id_:
ids = get_existent_ids()
id_ = random_module.randint(0, N)
while id_ in ids:
id_ = random_module.randint(0, N)
with open(DIRECTORY_PREDICT.joinpath(f'article{id_}.txt'),
'w',
encoding='utf-8') as f:
f.write(full_text)
text = overwrite_one_article(id_, directory=DIRECTORY_PREDICT)
my_predict_dataset = PropDataset(DIRECTORY_PREDICT, is_test=True)
my_predict_iter = data.DataLoader(dataset=my_predict_dataset,
batch_size=BATCH_SIZE,
shuffle=False,
num_workers=1,
collate_fn=pad)
tmp_file = 'tmp.txt'
eval(model,
my_predict_iter,
tmp_file,
criterion,
binary_criterion,
NUM_TASK=NUM_TASK)
ids, texts = read_data(DIRECTORY_PREDICT, is_test=True)
t_texts = clean_text(texts, ids)
flat_texts = [sentence for article in t_texts for sentence in article]
fi, prop_sents = convert(NUM_TASK - 1, flat_texts, tmp_file)
prop_sents = prop_sents[id_]
prop_sents = ['1' if elem else '' for elem in prop_sents]
results = remove_duplicates(fi)
DIRECTORY_PREDICT.joinpath(f'article{id_}.txt').rename(
DIRECTORY_MARKUP.joinpath(f'article{id_}.txt'))
lst = [set() for _ in range(len(full_text))]
source_lst = [set() for _ in range(len(full_text))]
for inner_lst in results:
for i in range(inner_lst[-2], inner_lst[-1]):
lst[i].add(HUMAN_READABLE_TECHNIQUES[TECHNIQUES.index(
inner_lst[-3])])
source_lst[i].add(inner_lst[-3])
extracts_s_e = []
extracts = []
categories = []
for elem in fi:
if elem[0] != str(id_):
continue
_, category, start, end = elem
extracts_s_e.append((start, end))
extracts.append(text[start:end])
categories.append(category)
extracts = [
' '.join(normalize(extract.strip())) for extract in extracts if extract
]
print(f'extracts: {extracts}')
# CHECK
# extracts = [word for sent in extracts for word in sent.split()]
test_x, test_maxlen = get_data(extracts,
vocab_size=args.vocab_size,
maxlen=args.maxlen)
test_x = sequence.pad_sequences(test_x,
maxlen=max(train_maxlen, test_maxlen))
test_length = test_x.shape[0]
splits = []
for i in range(1, test_length // args.batch_size):
splits.append(args.batch_size * i)
if test_length % args.batch_size:
splits += [(test_length // args.batch_size) * args.batch_size]
test_x = np.split(test_x, splits)
with graph.as_default():
aspect_model = keras_load_model(os.path.join('flask_app', 'output',
'reviews', 'model_param'),
custom_objects={
"Attention": Attention,
"Average": Average,
"WeightedSum": WeightedSum,
"MaxMargin": MaxMargin,
"WeightedAspectEmb":
WeightedAspectEmb,
"max_margin_loss":
U.max_margin_loss
},
compile=True)
test_fn = K.function([
aspect_model.get_layer('sentence_input').input,
K.learning_phase()
], [
aspect_model.get_layer('att_weights').output,
aspect_model.get_layer('p_t').output
])
aspect_probs = []
for batch in tqdm(test_x):
_, cur_aspect_probs = test_fn([batch, 0])
aspect_probs.append(cur_aspect_probs)
aspect_probs = np.concatenate(aspect_probs)
label_ids = np.argsort(aspect_probs, axis=1)[:, -5:]
for i, labels in enumerate(label_ids):
print(
f'{extracts[i]}: {[aspects[label] for label in labels][::-1]}')
correct_lst = ['; '.join(list(elem)) for elem in lst]
commands = {
extract: ([aspects[label] for label in label_ids[i]][::-1], [])
for i, extract in enumerate(extracts)
}
write_existent_dict(id_, source_lst, directory=DIRECTORY_MARKUP)
for f in glob.glob(f'{DIRECTORY_PREDICT}/*'):
os.remove(f)
return jsonify(
result={
'id': id_,
'list': correct_lst,
'text': text,
'prop_sents': prop_sents,
'commands': commands
})
# vehicle_dataset = 'data/vehicle-detector/voc.data'.encode("utf-8")
lp_threshold = .6
wpod_net_path = "data/lp-detector/wpod-net_update1.h5"
wpod_net = load_model(wpod_net_path)
ocr_threshold = .6
ocr_weights = 'data/ocr/ocr-net.weights'.encode("utf-8")
ocr_netcfg = 'data/ocr/ocr-net.cfg'.encode("utf-8")
ocr_dataset = 'data/ocr/ocr-net.data'.encode("utf-8")
ocr_net = dn.load_net(ocr_netcfg, ocr_weights, 0)
ocr_meta = dn.load_meta(ocr_dataset)
car_cat_threshold = .75
car_cat_net_path = "data/car-cat-detector/car-cat-detector.h5"
car_cat_net = keras_load_model(car_cat_net_path)
car_cat_img_size = 299
car_cat_vehicle_width_threshold = 120 # pixels
car_cat_vehicle_height_threshold = 120 # pixels
car_cat_names = []
with open('data/car-cat-detector/car-cat-detector.names') as csvDataFile:
csvReader = csv.reader(csvDataFile, delimiter=';')
for row in csvReader:
car_cat_names.append(row[0])
print(car_cat_names)
CAR_TRACK_MIN_DISTANCE = 40
CAR_TRACK_N_FRAME = 8