def create_global_step(session: tf.Session) -> tf.Variable:
"""
Creates the Tensorflow 'global_step' variable (see `MonitorContext.global_step_tensor`).
:param session: Tensorflow session the optimiser is running in
:return: The variable tensor.
"""
global_step_tensor = tf.Variable(0, trainable=False, name="global_step")
session.run(global_step_tensor.initializer)
return global_step_tensor
def create_detector(self, verbose, mtcnn_kwargs):
""" Create the mtcnn detector """
self.verbose = verbose
if self.verbose:
print("Adding MTCNN detector")
self.kwargs = mtcnn_kwargs
mtcnn_graph = Graph()
with mtcnn_graph.as_default():
mtcnn_session = Session()
with mtcnn_session.as_default():
pnet, rnet, onet = create_mtcnn(mtcnn_session, self.data_path)
mtcnn_graph.finalize()
self.kwargs["pnet"] = pnet
self.kwargs["rnet"] = rnet
self.kwargs["onet"] = onet
self.initialized = True
def update(self, sess: tf.Session, batch_observ, batch_next_observ,
batch_action):
feed_dict = {
self._observ: batch_observ,
self._next_observ: batch_next_observ,
self._action: batch_action,
}
_, loss, global_step, summary = sess.run([
self._train_op, self._train_loss,
tf.train.get_global_step(), self._train_summaries
],
feed_dict=feed_dict)
self._train_summary_writer.add_summary(summary,
global_step=global_step)
return loss
def __init__(self,
n_iterations,
Xt,
enable_dann,
do_strength=10,
batch_size=128,
plot=False,
mixup=False):
self.n_iterations = n_iterations
self.Xt = Xt
self.graph = Graph()
self.enable_dann = enable_dann
with self.graph.as_default():
self.session = Session()
with self.session.as_default():
self.model, self.source_classification_model, self.domain_classification_model, self.embeddings_model, self.domain_classification_model_stand_alone = build_models(
do_strength)
self.do_strength = do_strength
self.batch_size = batch_size
self.plot = plot
self.mixup = mixup
self.models = self.model, self.source_classification_model, self.domain_classification_model, self.embeddings_model, self.domain_classification_model_stand_alone
def train_model(sess: tf.Session, x, y, train_op, dataset, loss_op=None):
for e in range(EPOCHS):
avg_cost = 0.
total_batch = int(dataset.train_data.shape[0] / BATCH_SIZE)
for i in range(int(total_batch) - 1):
batch_start = i * BATCH_SIZE
batch = dataset.train_data[batch_start:batch_start + BATCH_SIZE]
labels = dataset.train_labels[batch_start:batch_start + BATCH_SIZE]
_, c = sess.run([train_op, loss_op],
feed_dict={
x: batch,
y: labels
})
avg_cost += c / total_batch
print("Epoch:", '%04d' % (e + 1), "cost={:.9f}".format(avg_cost))
def _metric_step(self, stage, initial_ops, sess: tf.Session, epoch: int, step=None, repeats=1, summary_every=1):
ops = initial_ops
offsets, lengths = [], []
trainers = self.active()
for trainer in trainers:
offsets.append(len(ops))
metric_ops = trainer.metric_ops(stage)
lengths.append(len(metric_ops))
ops.extend(metric_ops)
if repeats > 1:
all_results = np.stack([np.array(sess.run(ops)) for _ in range(repeats)])
results = np.mean(all_results, axis=0)
else:
results = sess.run(ops)
if step is None:
step = results[0]
for trainer, offset, length in zip(trainers, offsets, lengths):
chunk = results[offset: offset + length]
summaries = trainer.process_metrics(stage, chunk, epoch, step)
if trainer.summary_writer and step > 200 and (step % summary_every == 0):
summary = tf.Summary(value=summaries)
trainer.summary_writer.add_summary(summary, global_step=step)
return results
def load(self, session: tf.Session):
"""
## Load model as a set of numpy arrays
"""
checkpoints_path = pathlib.Path(self.path)
if not checkpoints_path.exists():
return False
max_step = -1
for c in checkpoints_path.iterdir():
max_step = max(max_step, int(c.name))
if max_step < 0:
return False
checkpoint_path = checkpoints_path / str(max_step)
with open(str(checkpoint_path / "info.json"), "r") as f:
files = json.loads(f.readline())
# Load each variable
for variable in self.__variables:
file_name = files[variable.name]
value = np.load(str(checkpoint_path / file_name))
ph = tf.placeholder(
value.dtype,
shape=value.shape,
name=f"{tf_util.strip_variable_name(variable.name)}_ph")
assign_op = tf.assign(variable, ph)
session.run(assign_op, feed_dict={ph: value})
self.max_step = max_step
return True
def __init__(self, projectModel, latentDim, usePretrained=True):
from tensorflow import GPUOptions, Session, ConfigProto
gpu_options = GPUOptions(per_process_gpu_memory_fraction=0.25)
sess = Session(config=ConfigProto(gpu_options=gpu_options))
self.latentDim = latentDim
self.actionSpace = projectModel.getActionSpace()
if usePretrained:
name = projectModel.modelOptions.asPretrainedVAE_Filename(
latentDim)
self.model = load_model(name)
else:
self.defineModel()
self.defineDecoder()
def _embedding_from_line(sess : tf.Session, nodes : Tuple[tf.placeholder, tf.placeholder], mel_dir : str, l : trainline) -> np.ndarray :
mel_node, gv_node = nodes
# load the mel-spectrogram
mel = np.load(os.path.join(mel_dir, l.mel))
gvector = sess.run(
gv_node,
{
mel_node : mel[None]
}
)
return gvector[0]
def train_step(self, sess: tf.Session, data, step_config):
if self.num_classes() == 2:
y = data["y"]
else:
y = data["label"]
feed_dict = {
self.x: data["x"],
self.expected_y: y,
self.is_training: True,
self.training_rate: step_config["training_rate"],
self.l2_loss_factor: step_config["l2_loss"],
}
fetch_dict = {"trainer": self.trainer, "summary": self.train_summary}
return sess.run(fetch_dict, feed_dict)
def __init__(self,
num_model_path,
char_model_path,
certainty_thresh=0.7,
multithreaded=True):
"""
Initialises PlateReader object
@param: num_model_path - path to keras ML model which IDs numbers
@param: num_model_path - path to keras ML model which IDs letters
"""
thread_graph = Graph()
with thread_graph.as_default():
self.thread_session = Session()
with self.thread_session.as_default():
self.num_model = keras.models.load_model(num_model_path)
self.char_model = keras.models.load_model(char_model_path)
self.graph = tf.get_default_graph()
self.parking_license_pairs = {}
self.certainty_thresh = certainty_thresh
self.plate_isolator = PlateIsolator()
self.letter_isolator = LetterIsolator()
def _metric_step(self, stage, initial_ops, sess: tf.Session, epoch: int, step=None, repeats=1, summary_every=1):
ops = initial_ops
offsets, lengths = [], []
trainers = self.active()
for trainer in trainers:
offsets.append(len(ops))
metric_ops = trainer.metric_ops(stage)
lengths.append(len(metric_ops))
ops.extend(metric_ops)
if repeats > 1:
all_results = np.stack([np.array(sess.run(ops)) for _ in range(repeats)])
results = np.mean(all_results, axis=0)
else:
results = sess.run(ops)
if step is None:
step = results[0]
for trainer, offset, length in zip(trainers, offsets, lengths):
chunk = results[offset: offset + length]
summaries = trainer.process_metrics(stage, chunk, epoch, step)
if trainer.summary_writer and step > 200 and (step % summary_every == 0):
summary = tf.Summary(value=summaries)
trainer.summary_writer.add_summary(summary, global_step=step)
return results
def __init__(self, model_func=None, feed_list_model=None, n_users=2):
self.model_func = model_func
self.feed_list_model = feed_list_model
self.n_users = n_users
self.graph = Graph()
with self.graph.as_default():
self.sess = Session()
K.set_session(self.sess)
def GetNext(self, sess: tf.Session = None):
"""
get next batch of images and labels
Args:
sess: a tf.Session
Returns:
images, labels
"""
if sess is None:
sess = tf.get_default_session()
image, label = sess.run(self._next)
return image, label
def do_eval(sess: tf.Session, eval_correct, images_placeholder,
labels_placeholder, data_set, settings: Settings) -> float:
true_count = 0 # Counts the number of correct predictions.
steps_per_epoch = data_set.num_examples // settings.batch_size
num_examples = steps_per_epoch * settings.batch_size
for step in range(steps_per_epoch):
feed_dict = fill_feed_dict(
data_set,
images_placeholder,
labels_placeholder,
settings,
)
true_count += sess.run(eval_correct, feed_dict=feed_dict)
precision = float(true_count) / num_examples
return precision
def translate_line(session: tf.Session,
line: str,
source_vocab: vocab.Vocabulary,
target_vocab: vocab.Vocabulary,
encoder_inputs: tf.Tensor,
decoder_inputs: tf.Tensor,
decoder_targets: tf.Tensor,
decoder_logits: tf.Tensor) -> str:
"""
Translates one single input string.
"""
source_ids = np.array(source_vocab.get_ids(line.split())).reshape(1, -1)
translated_ids = [] # type: List[int]
for _ in range(C.TRANSLATION_MAX_LEN):
# target ids will serve as decoder inputs and decoder targets,
# but decoder targets will not be used to compute logits
target_ids = np.array([C.BOS_ID] + translated_ids).reshape(1, -1)
feed_dict = {encoder_inputs: source_ids,
decoder_inputs: target_ids,
decoder_targets: target_ids}
logits_result = session.run([decoder_logits], feed_dict=feed_dict)
# first session result, first item in batch, target symbol at last position
next_symbol_logits = logits_result[0][0][-1]
next_id = np.argmax(next_symbol_logits)
if next_id in [C.EOS_ID, C.PAD_ID]:
break
if next_id == C.UNK_ID:
max = 0
for number in next_symbol_logits:
if number> max:
max = number
else:
pass
next_id = max
translated_ids.append(next_id)
words = target_vocab.get_words(translated_ids)
return ' '.join(words)
def test_first():
'''构建一个只有两个constant做输入, 然后进行矩阵乘的简单图:'''
# 如果不使用with session()语句, 需要手动执行session.close().
# with device设备指定了执行计算的设备:
# "/cpu:0": 机器的 CPU.
# "/gpu:0": 机器的第一个 GPU, 如果有的话.
# "/gpu:1": 机器的第二个 GPU, 以此类推.
with Session() as sess: # 创建执行图的上下文
with device('/cpu:0'): # 指定运算设备
mat1 = constant([[3, 3]]) # 创建源节点
mat2 = constant([[2], [2]])
product = matmul(mat1, mat2) # 指定节点的前置节点, 创建图
result = sess.run(product) # 执行计算
print(result)
def predict_batch(
sess: tf.Session,
frame_feat: tf.placeholder,
frame_len: tf.placeholder,
frame_temp: tf.placeholder,
frame_prob: tf.placeholder,
batch_frame_feat: np.array,
batch_frame_len: np.array,
):
feed_dict = {
frame_feat: batch_frame_feat,
frame_len: batch_frame_len,
frame_temp: 0.9
}
batch_frame_prob = sess.run(frame_prob, feed_dict=feed_dict)
return batch_frame_prob
def forward_inverse(
self,
sess: tf.Session,
flow: fl.FlowLayer,
inputs: fl.FlowData,
feed_dict=None,
atol: float = 1e-6,
):
x_input = inputs
y = flow(x_input, forward=True)
x_rec = flow(y, forward=False)
for c, cprim in zip(x_rec, x_input):
if type(c) == tf.Tensor and type(cprim) == tf.Tensor:
c_np, cprim_np = sess.run([c, cprim], feed_dict=feed_dict)
self.assertAllClose(c_np, cprim_np, atol=atol)
def predict(self, session: tf.Session, left_imgs, right_imgs):
"""
测试
:param session: tf.session
:param left_imgs: 左侧视图batch
:param right_imgs: 右侧视图batch
:return: prediction
"""
prediction = session.run(
self.disparity_3,
feed_dict={
self.left_inputs: left_imgs,
self.right_inputs: right_imgs,
self.is_training: True # 这里仍然需要设置为true bn的问题需要确认
})
return prediction
class KerasClassifier(ClassifierABC):
def __init__(self, model: Model, trainer: KerasTrainer):
self.model = model
self.trainer: Optional[KerasTrainer] = trainer
def fit(self, images: List[Image], labels: List[int], validation_size: int) -> "KerasClassifier":
assert self.trainable, "You can't train an un-pickled classifier"
images = asarray(images)
labels = to_categorical(asarray(labels), self.n_classes)
train_images, train_labels = images[:-validation_size], labels[:-validation_size]
val_images, val_labels = images[-validation_size:], labels[-validation_size:]
self.trainer.train(self.model, train_images, train_labels, val_images, val_labels)
return self
def predict_proba(self, examples: List[Image]) -> Sequence[float]:
try: # FIXME
with self.graph.as_default(), self.session.as_default():
return self.model.predict(asarray(examples))
except AttributeError:
return self.model.predict(asarray(examples))
def __getstate__(self) -> Dict:
with NamedTemporaryFile(suffix=".hdf5", delete=True) as fd:
self.model.save(fd.name, overwrite=True, include_optimizer=False)
model_str = fd.read()
state = copy(self.__dict__)
state.pop("model")
state.pop("trainer")
return {**state, "model_str": model_str}
def __setstate__(self, state: Dict):
self.__dict__.update(state)
self.graph = Graph()
with NamedTemporaryFile(suffix=".hdf5", delete=True) as fd:
fd.write(state.pop("model_str"))
fd.flush()
with self.graph.as_default():
self.session = Session(graph=self.graph)
with self.session.as_default():
self.model = load_model(fd.name, compile=False)
self.trainer = None
@property
def trainable(self) -> bool:
return self.trainer is not None
def _parse_graph_info(graph_def):
"""Parse GraphDef
Fetch input tensors and output tensors name for reconstructing
graph in uTensor Context object
Argument
========
- graph_def <tf.GraphDef>: a GraphDef object
Return
======
- graph_nodes <defaultdict>: a dict with key as operation name and
value as a defaultdict with keys 'input_tensor' and 'output_tensor'
which maps to a set of input/output tensor names respectively
Note
====
- thought the output tensor names is irrelevent for TensorFlow, but it
is neccessary for uTensor
"""
OperationInfo = namedtuple('OperationInfo',
field_names=[
'input_tensor', 'output_tensor', 'op_type',
'output_content', 'op_attr'
])
graph = Graph()
with graph.as_default(): # pylint: disable=E1129
import_graph_def(graph_def, name="")
graph_info = {}
with Session(graph=graph):
for node in graph_def.node:
op = graph.get_operation_by_name(node.name)
input_tensor = [(t.name, t.dtype, _parse_shape(t.shape))
for t in op.inputs]
output_tensor = [(t.name, t.dtype, _parse_shape(t.shape))
for t in op.outputs]
op_type = node.op
output_content = {}
op_attr = node.attr
if node.op in ["Const"]:
for tensor_name, _, _ in output_tensor:
output_content[tensor_name] = make_ndarray(
node.attr['value'].tensor)
graph_info[node.name] = OperationInfo(input_tensor, output_tensor,
op_type, output_content,
op_attr)
return graph_info
class FaceID:
def __init__(self):
self.to_identify_path = os.path.join(os.path.abspath(os.sep), 'ALFI_Data','To_Identify', 'Raw')
self.to_identify_processed_path = os.path.join(os.path.abspath(os.sep), 'ALFI_Data','To_Identify', 'Processed')
self.to_process_path = os.path.join(os.path.abspath(os.sep), 'ALFI_Data','To_Process')
self.dataset_path = os.path.join(os.path.abspath(os.sep), 'ALFI_Data','Dataset','DC')
self.dataset_ds_path = os.path.join(os.path.abspath(os.sep), 'ALFI_Data','Dataset','DS')
self.weigths_path = os.path.join(os.path.abspath(os.sep), 'ALFI_Data','Weigths')
self.current_sbj = 0
while(os.path.exists(os.path.join(self.dataset_path,'sbj-'+str(self.current_sbj)))):
self.current_sbj = self.current_sbj + 1
self.graph = Graph()
with self.graph.as_default():
self.session = Session()
with self.session.as_default():
self.model = faceIDNet()
def train(self, epochs, save_name, load=False):
if load:
self.load(save_name)
K.set_session(self.session)
with self.graph.as_default():
gen = generator(24, self.dataset_ds_path)
save_folder = os.path.join(self.weigths_path, save_name)
if not os.path.exists(save_folder):
os.makedirs(save_folder)
cp_callback = ModelCheckpoint(os.path.join(save_folder, 'faceID_weights'), save_weights_only=True)
self.model.fit_generator(gen, steps_per_epoch=30, epochs=epochs, validation_steps=20, callbacks=[cp_callback])
lossTrain = self.model.evaluate_generator(gen, steps=30)
print('* - Loss: '+str(lossTrain))
def predict(self, inputs, threshold=0.2):
K.set_session(self.session)
with self.graph.as_default():
inputs = [inputs[0,:].reshape((1,100,100,4)), inputs[1,:].reshape((1,100,100,4))]
out = self.model.predict(inputs)
return (out <= threshold)
def load(self, save_name):
K.set_session(self.session)
with self.graph.as_default():
self.model.load_weights(os.path.join(self.weigths_path, save_name, 'faceID_weights'))
self.model._make_predict_function()
print('--- Weights loaded ---')
def _embedding_from_line_direct(sess : tf.Session, nodes : Tuple[tf.placeholder, tf.placeholder],p_mel : str) -> np.ndarray :
mel_node, gv_node = nodes
# load the mel-spectrogram
# mel = np.load(p_mel).T # when used for generated mel
mel = np.load(p_mel) # when used for extracted mel
# the difference being axis order
gvector = sess.run(
gv_node,
{
mel_node : mel[None]
}
)
return gvector[0]
def evaluate_generator_output(sess: tf.Session, real_batch: np.ndarray,
z_dim: int, generator_input: tf.Tensor,
layer_sizes: List[int],
show_samples: bool) -> type(None):
"""
Print sets created by the generator and evaluate their
plausibility.
:param sess:
session with (partially) trained generator
:param real_batch:
real batch such that sets to be generated are conditioned
on conditions from it
:param z_dim:
number of dimensions (features) of random noise that is passed
to generator
:param generator_input:
placeholder for generator input
:param layer_sizes:
sizes of layers of the network
:param show_samples:
if `True`, conditions and generated sets are printed
:return:
None
"""
generator_batch = training_set.turn_into_generator_batch(real_batch, z_dim)
n_items = real_batch.shape[1] // 2
conditions = real_batch[:, :n_items].astype(np.int32)
samples = sess.run(generator(generator_input,
n_items,
layer_sizes,
is_applied=True),
feed_dict={generator_input: generator_batch})
binary_samples = np.around(samples).astype(np.int32)
score = 0
for i in range(samples.shape[0]):
if show_samples:
print(f"Condition: {conditions[i, :].tolist()}")
print(f"Binary sample: {binary_samples[i, :].tolist()}")
print(f"Sample: {samples[i, :].tolist()}\n")
if (conditions[i, :] - binary_samples[i, :]).max() == 0:
score += 1
score /= samples.shape[0]
print(f"Plausibility score is {score}.\n\n")
return score
def estimate(self, sess: tf.Session, rewards, terminals, observs, actions):
"""Estimate target value
Args:
sess: tf.Session
rewards: batch of rewards
terminals: batch of terminals(done)
actions: batch of actions
"""
feed_dict = {
self._reward: rewards,
self._terminal: terminals,
# For calculate next value
self._observ: observs,
self._action: actions,
}
return sess.run(self.y, feed_dict=feed_dict)
def read_next(self, sess: tf.Session):
with self._update_lock:
fetches = [self._read_op, self._size_op,
self._footage.frames[0],
self._footage.features[0],
self._footage.labels[0],
self._footage.true_steer[0]]
step, size, frame_file, feature_file, label_file, _ \
= sess.run(fetches)
step = (step - 1) % len(self._footage.modes)
mode = self._footage.modes[step]
timestamp = str(datetime.datetime.now().ctime())
print('[%s] refresh [%d] [%d] [%s] [%s] [%s]'
% (timestamp, size, mode, frame_file,
feature_file, label_file))
self._total_frames = size
self._mode = mode
def detect_single_img(sess: tf.Session,
tensor_dict: Mapping[str, tf.Tensor],
image_tensor: tf.Tensor,
img_path: str) -> Dict[str, Any]:
"""Runs detector on a single image.
Args:
sess: tf.Session
tensor_dict: dict, maps from str name to tf.Tensor in graph, keys
include ['num_detections', 'detection_boxes', 'detection_scores',
'detection_classes']
image_tensor: tf.Tensor, shape [batch_size, H, W, C]
img_path: str, path to image file
Returns: dict, maps keys from tensor_dict to results
Raises:
FileNotFoundError, if img_path is invalid
TypeError, if image is not uint8
"""
global images_missing
# Skip images that we do not have available right now
# - this is useful for processing parts of large datasets
if not os.path.isfile(img_path):
if not images_missing:
print('Suppressing further warnings about missing files.')
images_missing = True
raise FileNotFoundError(f'Could not find: {img_path}')
# Load image
image = np.array(Image.open(img_path))
if image.dtype != np.uint8:
raise TypeError(f'Image was not type np.uint8: {img_path}')
# Run inference
image = image[np.newaxis, :] # add batch dimension
output_dict = sess.run(tensor_dict, feed_dict={image_tensor: image})
# remove batch dimension, and convert from float32 to appropriate type
for key in output_dict:
output_dict[key] = output_dict[key][0]
output_dict['num_detections'] = int(output_dict['num_detections'])
output_dict['detection_classes'] = output_dict['detection_classes'].astype(
np.uint8)
return output_dict
def test(session: tf.Session, loss_value, accuracy_value, batches):
with logger.section("Test", total_steps=batches):
test_loss = 0
correct = 0
batch_idx = -1
while True:
batch_idx += 1
try:
l, a = session.run([loss_value, accuracy_value])
test_loss += l
correct += a
except tf.errors.OutOfRangeError:
break
logger.progress(batch_idx + 1)
logger.store(test_loss=test_loss / batches)
logger.store(accuracy=correct / batches)
def perform_segmentation(input_file: str,
network: tf.Tensor,
lesion_image: tf.Tensor,
_: tf.Tensor,
session: tf.Session,
output_directory: Optional[str] = None):
output_file = get_segmentation_file_name(input_file, output_directory)
if os.path.exists(output_file):
log.warning(
f"Output file {output_file} already exists, will be ignored")
return
network_input = load_transform_lesion(input_file)
segmentation = session.run(network,
feed_dict={lesion_image: network_input})
save_segmentation(segmentation, output_file)
log.info(f"Saved segmentation of {input_file} to {output_file}")
def _eval_threshold_for_one_node(output_node: tf.Tensor,
input_node: Union[tf.Tensor,
str], sess: tf.Session,
batch_size: int) -> Tuple[float, float]:
res_min = [np.Inf for i in range(output_node.shape[3].value)]
res_max = [-np.Inf for i in range(output_node.shape[3].value)]
print('Eval initial threshold:', output_node.name)
for i in range(int(10000 / batch_size) + 1):
res = sess.run(output_node)
res_min = np.minimum(
res_min, np.min(np.min(np.min(res, axis=0), axis=0), axis=0))
res_max = np.maximum(
res_max, np.max(np.max(np.max(res, axis=0), axis=0), axis=0))
return res_min, res_max
def evaluate(self, sess: tf.Session, inputs, target_results):
metric_values_tensors = []
metric_update_ops = []
for value_tensor, update_op in self.metrics_array.values():
metric_values_tensors.append(value_tensor)
metric_update_ops.append(update_op)
results = sess.run(
metric_values_tensors + metric_update_ops,
feed_dict={
self.picks_dropout: 1.0,
self.picks_inputs: inputs,
self.picks_target_results: target_results
})
metric_values = results[:len(metric_values_tensors)]
return {name: value for name, value in zip(self.metrics_array.keys(), metric_values)}
def _dojob(ready, e, queue):
prctl.set_name('AI detector - do job')
global session1, session2, ip_model, mac_model
ip_graph = Graph()
config = ConfigProto()
config.gpu_options.allow_growth = True
with ip_graph.as_default():
session1 = Session(config=config)
with session1.as_default():
ip_model = K.models.load_model(
'gru_ip_4tuple.hdf5', custom_objects={'attention': attention})
ip_model._make_predict_function()
mac_graph = Graph()
with mac_graph.as_default():
session2 = Session(config=config)
with session2.as_default():
mac_model = K.models.load_model(
'gru_mac_4tuple.hdf5', custom_objects={'attention': attention})
mac_model._make_predict_function()
ready.set()
print 'set ready'
last = time.time()
global ignore_packet
while e.is_set() == False:
if queue.empty() == False:
obj = queue.get()
if (obj[0], obj[1]) in ignore_packet:
if obj[3] <= ignore_packet[(obj[0], obj[1])]:
continue
feature_extract((obj[2], obj[3]))
if time.time() - last >= polling_interval:
print queue.qsize()
global flow_statics, src_addr_list, memory_data
# calculate features in last 5 seconds
result = calculate_feature(flow_statics)
memory_data.pop(0)
memory_data.append(result)
t_run_exp = threading.Thread(target=_run_exp,
args=(
result,
src_addr_list,
memory_data,
))
t_run_exp.start()
t_run_exp.join()
flow_statics = {}
src_addr_list = {}
last = time.time()
K.backend.clear_session()
del ip_model
del mac_model
class KerasModel(object):
"Load the Keras Model"
def __init__(self):
self.initialized = False
self.verbose = False
self.model_path = self.set_model_path()
self.model = None
self.session = None
@staticmethod
def set_model_path():
""" Set the path to the Face Alignment Network Model """
model_path = os.path.join(os.path.dirname(__file__),
".cache", "2DFAN-4.h5")
if not os.path.exists(model_path):
raise Exception("Error: Unable to find {}, "
"reinstall the lib!".format(model_path))
return model_path
def load_model(self, verbose, dummy, ratio):
""" Load the Keras Model """
if self.initialized:
return
self.verbose = verbose
if self.verbose:
print("Initializing keras model...")
keras_graph = Graph()
with keras_graph.as_default():
config = ConfigProto()
if ratio:
config.gpu_options.per_process_gpu_memory_fraction = ratio
self.session = Session(config=config)
with self.session.as_default():
self.model = keras.models.load_model(
self.model_path,
custom_objects={'TorchBatchNorm2D':
TorchBatchNorm2D})
self.model.predict(dummy)
keras_graph.finalize()
self.initialized = True
def load_model(self, verbose, dummy, ratio):
""" Load the Keras Model """
self.verbose = verbose
if self.verbose:
print("Initializing keras model...")
keras_graph = Graph()
with keras_graph.as_default():
config = ConfigProto()
if ratio:
config.gpu_options.per_process_gpu_memory_fraction = ratio
self.session = Session(config=config)
with self.session.as_default():
self.model = keras.models.load_model(
self.model_path,
custom_objects={'TorchBatchNorm2D':
TorchBatchNorm2D})
self.model.predict(dummy)
keras_graph.finalize()
self.initialized = True
def _create_dummy_variable(session: tf.Session):
dummy_var = tf.Variable(0, name='dummy_var', dtype=tf.int32)
session.run(tf.variables_initializer([dummy_var]))
return dummy_var