def __init__(self, ctrl, topo):
self.log = logger.getLogger("EzSegway", constants.LOG_LEVEL)
self.log.debug("Create EzSegway")
self.ctrl = ctrl
self.topo = topo
self.old_flows = []
self.new_flows = []
Simulation.Process.__init__(self, name='EzSegway')
def __init__(self):
self.execution_time_by_test_number = defaultdict()
self.rule_overheads = {}
self.execution_results = defaultdict()
self.cdf_time_using_new_path = defaultdict()
self.is_reading_deadlock_exe = False
self.log = logger.getLogger("tracer", constants.LOG_LEVEL)
self.number_of_rules_by_test_number = defaultdict()
self.message_overheads = {}
self.current_method = ''
self.current_topo = ''
self.counters = defaultdict()
def main(key):
'''action - checking if process exists,
starting if not, and logging the result anyway'''
if not findProcess('python scraper.py'):
date = datetime.now().strftime('%Y_%m_%d')
print '%s | deploying new scraper\n' % date
mailer.send_welcoming()
subprocess.call('python scraper.py &', shell=True)
else:
log = logger.getLogger(recent=True)
log.info('Checking: scraper is working')
print 'Checking: scraper is working'
def main(key):
'''action - checking if process exists,
starting if not, and logging the result anyway'''
if not findProcess('python scraper.py'):
date = datetime.now().strftime('%Y_%m_%d')
print '%s | deploying new scraper\n' % date
mailer.send_welcoming()
subprocess.call('python scraper.py &', shell=True)
else:
log = logger.getLogger(recent=True)
log.info('Checking: scraper is working')
print 'Checking: scraper is working'
def save_model(model_paths, model, target="", compact=False):
"""Save a model to given dir"""
from os import path
from os import makedirs
import tensorpack as tp
from tensorpack.tfutils.varmanip import get_checkpoint_path
from tensorpack.tfutils.export import ModelExporter
import misc.logger as logger
_L = logger.getLogger("Saver")
save_to_modeldir = target is ""
for model_path in model_paths:
# get model path
real_path = get_checkpoint_path(model_path)
abs_p = path.realpath(model_path)
if (not path.isfile(abs_p)):
_L.error("{} is not a model file".format(model_path))
continue
# save to same folder as model
if (save_to_modeldir):
target = path.dirname(abs_p)
# make sure the folder exists
if not path.exists(target):
makedirs(target)
conf = tp.PredictConfig(session_init=tp.get_model_loader(model_path),
model=model,
input_names=["input"],
output_names=["emb"])
exporter = ModelExporter(conf)
if (compact):
out = path.join(target, "{}.pb".format(path.basename(real_path)))
_L.info("saving {} to {}".format(path.basename(real_path), out))
exporter.export_compact(out)
else:
_L.info("compact saving {} to {}".format(path.basename(real_path),
target))
exporter.export_serving(target)
import itertools
import collections
from collections import deque
from misc import logger, global_vars
from misc.utils import *
from domain.network_premitives import *
from ez_lib.ez_ob import *
from copy import deepcopy, copy
log = logger.getLogger("segmentation", constants.LOG_LEVEL)
mulog = logger.getLogger("ez_flow", constants.LOG_LEVEL)
PathDiffResult = collections.namedtuple("PathDiffResult", ["id", "to_same", "segs"])
def split_flows(old_flows, new_flows):
log.debug("inside split_flows")
current_id = len(old_flows) - 1
additional_flows = []
for old_flow, new_flow in itertools.izip(old_flows, new_flows):
if old_flow.path == [] and new_flow.path != []:
old_flow.update_type = constants.ADDING_FLOW
new_flow.update_type = constants.ADDING_FLOW
elif new_flow.path == [] and old_flow.path != []:
old_flow.update_type = constants.REMOVING_FLOW
new_flow.update_type = constants.REMOVING_FLOW
elif old_flow.path != [] and new_flow.path != []:
s_flow = __split_flow__(old_flow, new_flow, current_id)
if s_flow is not None:
current_id = s_flow.flow_id
additional_flows.append(s_flow)
type=str,
default=constants.TOPO_ROCKETFUEL)
parser.add_argument('--method',
nargs='?',
type=str,
default=constants.P2P_METHOD)
parser.add_argument('--generating_method',
nargs='?',
type=str,
default=constants.LINK_FAILURE_GENERATION)
parser.add_argument('--test_number', nargs='?', type=int, default=10)
parser.add_argument('--number_of_flows', nargs='?', type=int, default=1000)
parser.add_argument('--failure_rate', nargs='?', type=float, default=0.5)
args = parser.parse_args()
directory = "../%s" % (args.logFolder)
if not os.path.exists(directory):
os.makedirs(directory)
# numeric_level = getattr(logging, args.logLevel.upper(), None)
# if not isinstance(numeric_level, int):
# raise ValueError('Invalid log level: %s' % numeric_level)
# constants.LOG_LEVEL = numeric_level
logger.init("../" + args.logFolder + "/" + args.logFile,
constants.LOG_LEVEL)
log = logger.getLogger("ez-segway", constants.LOG_LEVEL)
log.info("---> Log start <---")
run_experiment(args, log)
def init_logger():
return logger.getLogger("ControllerHandler", constants.LOG_LEVEL)
HEIGHT = uint32 ;
CHANNELS = uint32 ;
COUNT = uint32 ;
- msg header : <messageID:uint?>:
"""
import os
import io # noqa
import cnn.modules.term as term
from cnn.modules.config import Conf
from cnn import config
import misc.logger as logger
_L = logger.getLogger("CNNmain")
def argslice(s):
a = [int(e) if e.strip() else None for e in s.split(":")]
return slice(*a)
def setup_argparser(conf=None):
"""Setup the ArgumentParser for command line args.
Arguments:
config - dict-like object with default values"""
import argparse
import textwrap
if not conf:
def init_logger():
return logger.getLogger("flow_generator", constants.LOG_LEVEL)
def init_logger():
print("create logger for path_generator")
return logger.getLogger("path_generator", constants.LOG_LEVEL)
from cnn.resnet import (
preresnet_group, preresnet_basicblock, preresnet_bottleneck,
resnet_group, resnet_basicblock, resnet_bottleneck, se_resnet_bottleneck,
resnet_backbone)
from cnn.modules.dataflow import LineData, MyPrefetchDataZMQ, StripData, MyTestDataSpeed # noqa
from cnn.modules.npy_dataflow import LineData as LineDataNPZ
from cnn.modules.imagereader import LineData as LineDataImageReader
from cnn.client import Client
import cnn.misc as cm
from misc import logger
# for line profiling
# from cnn import profile
_L = logger.getLogger("CNN")
# _L.setLevel(logger.DEBUG)
# from cnn.misc import start_timer, pause_timer, print_times
abort_after_epoch = False
class AbortCallback(tp.Callback):
def _trigger_epoch(self):
if abort_after_epoch:
_L.info("Aborted by user.")
raise tp.StopTraining()
class ROCValidationCallback(tp.Callback):
'''
Adds self.validation_predictor (tp.OnlinePredictor) and self.validation_dataset (tp.DataFlow)
# import threading
import socket
from struct import pack, unpack
# from functools import wraps
import ctypes
from misc.logger import getLogger, DEBUG
_L = getLogger("client")
_L.setLevel(DEBUG)
class ClientCls:
class Answer:
def __init__(self, data=b"", error=None):
self.data = data
self.error = error
magic = ctypes.c_uint32(0x9E2B83C1).value
pack_magic = pack("!I", magic)
def __init__(self, ip="0.0.0.0", port=9645):
self.ip = ip
self.port = port
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.is_connected = False
self.msg_count = 0
def __pack_msg(self, fmt, val):
def init_logger(id_):
# logger.init('/root/ez-segway/logs/handler/handlerlog%d'% id_,constants.LOG_LEVEL)
return logger.getLogger("SwitchHandler%d" % id_, constants.LOG_LEVEL)
def init_logger():
return logger.getLogger("Centralized Controller", constants.LOG_LEVEL)
def init_logger(id_):
return logger.getLogger("Switch%d" % id_, constants.LOG_LEVEL)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='ez-segway sim.')
parser.add_argument('--logFolder', nargs='?', type=str, default="logs")
parser.add_argument('--logFile', nargs='?', type=str, default="stdout")
parser.add_argument('--data_file', nargs='?', type=str, default="data")
parser.add_argument('--end_flow', nargs='?', type=str, default="data")
args = parser.parse_args()
directory = "../%s" % (args.logFolder)
if not os.path.exists(directory):
os.makedirs(directory)
logger.init("../" + args.logFolder + "/" + args.logFile,
constants.LOG_LEVEL)
log = logger.getLogger("data-generator", constants.LOG_LEVEL)
log.info("---> Log start <---")
flow_gen = FlowChangeGenerator()
filename = "../%s" % args.data_file
update = flow_gen.read_flows(filename)
for flow in update.old_flows:
flow.path = []
flow_gen.write_flows_pair(filename, update.old_flows, update.new_flows)
filename = "../%s" % args.end_flow
update = flow_gen.read_flows(filename)
for flow in update.new_flows:
flow.path = []
flow_gen.write_flows_pair(filename, update.old_flows, update.new_flows)
import itertools
from ez_lib import ez_flow_tool
from collections import defaultdict
from ez_scheduler import EzScheduler
from ez_lib.ez_ob import CenUpdateInfo, UpdateNext
from misc import constants, logger
from domain.message import *
from collections import deque
from misc import global_vars
import time
import eventlet
mulog = logger.getLogger('cen_scheduler', constants.LOG_LEVEL)
class CenCtrlScheduler(EzScheduler):
def __init__(self, switches_, log_):
self.switches = switches_
super(CenCtrlScheduler, self).__init__(0, log_)
self.remaining_vol_of_dependency_loop_on_link = {}
self.received_updated_msg = defaultdict()
self.received_removed_msg = defaultdict()
########## Begin three properties are used for parallel processes ##########
self.no_of_pending_msgs = {}
self.notification_queues = {x: deque([]) for x in self.switches}
self.current_notification_time = {x: -1 for x in self.switches}
self.current_processing_time = {x: -1 for x in self.switches}
########### End three properties are used for parallel processes ###########
self.to_sames = defaultdict(list)
self.encounter_deadlock = False
import sys
import networkx as nx
import fnss
from misc import logger, constants
from sphere import Sphere
log = logger.getLogger("topology", constants.LOG_LEVEL)
class Topology:
def __init__(self, graph):
self.graph = graph
@staticmethod
def read_adjacency(filename):
"""Reads adjacency matrix from a file.
Returns:
list[list of bool]
"""
def IntLineReader(filename):
with open(filename) as f:
for line in f:
yield [bool(int(x)) for x in line.strip().split(' ')]
adjacency = list(IntLineReader(filename))
graph = nx.Graph()
src = 0
for row in adjacency:
# -*- coding: utf-8 -*-
import tensorpack as tp
import sys
from os import listdir, path
from cnn.modules.fio import FBytesIO
from cnn import config
import misc.logger as logger
_L = logger.getLogger("DataFlow")
def np_convert():
"""Create list -> np.array converter.
This is to cache the conversion dictionary
while keeping global namespace clean"""
import numpy as np
conv = {
int: np.int32,
bool: np.int32,
float: np.float32,
}
def asarray(el):
if not el:
raise AttributeError("Empty list not supported")
dt = conv.get(type(el[0]))
if not dt:
def test(dataset, model_paths, model=None):
import tensorpack as tp
import tensorflow as tf
from tensorpack.tfutils.varmanip import get_checkpoint_path
from tensorflow.contrib.tensorboard.plugins import projector
import numpy as np
import misc.logger as logger
import cnn.plotter as plotter
import struct
def pairwise(sess):
from cnn.main import pairwise_distance
descs = tf.compat.v1.placeholder(tf.float32, (
None,
8,
), "descs")
cnn_dists = pairwise_distance(descs, True)
pred = tp.OnlinePredictor([descs], [cnn_dists], sess=sess)
def calculate(*args, **kwargs):
return pred(*args, **kwargs)[0]
return calculate
_L = logger.getLogger("test") # noqa
sess = tf.compat.v1.Session()
cnn_summ = tf.compat.v1.summary.FileWriter(tp.logger.get_logger_dir() +
"/cnn")
lbd_summ = tf.compat.v1.summary.FileWriter(tp.logger.get_logger_dir() +
"/lbd")
cnn_logdir = cnn_summ.get_logdir()
# [[Match...], [Label...], [Descriptor...]]
dataset.reset_state()
data = fake_data(dataset, len(model_paths), _L)
pairwise_distance = pairwise(sess)
for model_path in model_paths:
# get global step
real_path = get_checkpoint_path(model_path)
reader = tf.compat.v1.train.NewCheckpointReader(real_path)
global_step = reader.get_tensor("global_step")
# predictor
pred = get_predictor(real_path, model)
img_info = [[], []]
imgs = []
# summaries
cnn_summaries = tf.compat.v1.Summary()
lbd_summaries = tf.compat.v1.Summary()
# collected data for ROC curves
roc_data = []
for batch_n in range(config.batch_num):
# test the batch
(emb, heights, labels, left, timgs, tinfo,
cnn_dists, lbd_dists) = test_batch(pairwise_distance, pred,
next(data), batch_n)
roc_data.append((labels, left, cnn_dists, lbd_dists))
# save results of CNN to NPZ file
# if config.save_results is not None:
# from pathlib import Path
#
# # convert to path and use NPZ suffix
# file = Path(str(config.save_results))
# file = file.with_suffix('.npz')
# file = str(file)
#
# # save NPZ to given path
# np.savez_compressed(file, cutout_list=list(timgs), cnn_desc_list=list(emb), cnn_dist_matrix=cnn_dists,
# label_list=list(labels), left_list=list(left), target_height_list=list(heights))
#
# _L.debug('Saved CNN results to \'{}\''.format(file))
if config.save_results:
nested = dataset.ds.ds
save_op = getattr(nested, 'save_results', None)
# check if nested dataset has a callable 'save_results' function
if callable(save_op):
nested.save_results(cnn_desc_list=list(emb),
label_list=list(labels),
left_list=list(left))
# TODO: warn user that save results is not yet implemented!
else:
pass
if config.return_results and hasattr(dataset,
'client') and dataset.client:
message = struct.pack('II', *cnn_dists.shape[:2])
message += cnn_dists.tobytes()
dataset.client.send('c', message, wait=False)
# generate image output
if config.tp_imgs:
cimgs = [
np.resize(el, (heights[i], *el.shape[1:]))
for i, el in enumerate(timgs)
]
imgs.append(cimgs)
for i in [0, 1]:
img_info[i].append(tinfo[i])
# generate projection output
if config.tp_proj:
mdata_name = "metadata_{}.tsv".format(batch_n)
mdata_path = "{}/{}".format(cnn_logdir, mdata_name)
with open(mdata_path, "w") as mfile:
for label in labels:
mfile.write("{}\n".format(label))
sprite = make_sprite(timgs, timgs[0].shape, cnn_logdir,
batch_n)
sprite_size = max(timgs[0].shape)
embv = tf.Variable(emb, name="embeddings")
initop = tf.variables_initializer([embv])
pconf = projector.ProjectorConfig()
embconf = pconf.embeddings.add()
embconf.tensor_name = embv.name
embconf.metadata_path = mdata_name
embconf.sprite.image_path = sprite
embconf.sprite.single_image_dim.extend(
[sprite_size, sprite_size])
projector.visualize_embeddings(cnn_summ, pconf)
sess.run(initop)
saver = tf.train.Saver()
saver.save(sess, "{}/embeddings.ckpt".format(cnn_logdir),
batch_n)
# generate ROC
tap_dists, tan_dists = split_dists(roc_data)
cnn_plots = plotter.plot_roc(tap_dists[0],
tan_dists[0],
"cnn: {}".format(config.depth),
color="g")
plotter.plot_roc(tap_dists[1],
tan_dists[1],
"lbd",
color="b",
figs=cnn_plots)
suffix = ""
for i in range(2):
img = plotter.plot_to_np(cnn_plots[i])
img = np.expand_dims(img, axis=0)
s = tp.summary.create_image_summary(
"ROC{}/{}".format(suffix, global_step), img)
cnn_summaries.value.extend(s.value)
suffix = "_zoomed"
# get images
# add image summary
if config.tp_imgs:
for i in range(len(img_info[0])):
for j in range(len(img_info[0][i])):
cnn_info = img_info[0][i][j]
lbd_info = img_info[1][i][j]
generate_tensorboard_img_summary(cnn_summaries, cnn_info,
imgs[i], "imgs/cnn",
global_step, i)
generate_tensorboard_img_summary(lbd_summaries, lbd_info,
imgs[i], "imgs/lbd",
global_step, i)
cnn_summ.add_summary(cnn_summaries, global_step)
lbd_summ.add_summary(lbd_summaries, global_step)
lbd_summ.flush()
cnn_summ.flush()
lbd_summ.close()
cnn_summ.close()
sess.close()
import itertools
import collections
from collections import deque
from misc import logger, global_vars
from misc.utils import *
from domain.network_premitives import *
from ez_lib.ez_ob import *
from copy import deepcopy, copy
log = logger.getLogger("segmentation", constants.LOG_LEVEL)
PathDiffResult = collections.namedtuple("PathDiffResult",
["id", "to_same", "segs"])
def split_flows(old_flows, new_flows):
log.debug("inside split_flows")
current_id = len(old_flows) - 1
additional_flows = []
for old_flow, new_flow in itertools.izip(old_flows, new_flows):
if old_flow.path == [] and new_flow.path != []:
old_flow.update_type = constants.ADDING_FLOW
new_flow.update_type = constants.ADDING_FLOW
elif new_flow.path == [] and old_flow.path != []:
old_flow.update_type = constants.REMOVING_FLOW
new_flow.update_type = constants.REMOVING_FLOW
elif old_flow.path != [] and new_flow.path != []:
s_flow = __split_flow__(old_flow, new_flow, current_id)
if s_flow is not None:
current_id = s_flow.flow_id
additional_flows.append(s_flow)
def visualize(model_path, model, iimgs):
import matplotlib.pyplot as plt
from os import path
from matplotlib.image import imread
import numpy as np
import tensorpack as tp
import misc.logger as logger
_L = logger.getLogger("visualize")
_L.debug(iimgs)
images = [imread(img) for img in iimgs]
imgs = [np.array([img]) for img in images]
# imgs = [np.expand_dims(img, 0) for img in imgs]
# print(imgs[0].shape)
# print(np.expand_dims(imgs[0], 0).shape)
# print(np.expand_dims(imgs[0], 3).shape)
# exit()
config.c_width = imgs[0].shape[2]
config.c_height = imgs[0].shape[1]
_L.debug("{} x {} cutouts".format(config.C_WIDTH, config.C_HEIGHT))
pred = tp.OfflinePredictor(
tp.PredictConfig(session_init=tp.get_model_loader(model_path),
model=model(config.depth, config.mode),
input_names=["input"],
output_names=["emb"]))
preds = [pred(el)[0] for el in imgs]
for i, p in enumerate(preds):
print("pred{}: ".format(i), end="")
print(p)
dists = [
np.sum((preds[0] - preds[i])**2, 1)[0] for i in range(1, len(preds))
]
for i, d in enumerate(dists):
print("dist{}: ".format(i), end="")
print(d)
file_name = path.basename(iimgs[0])
name_parts = file_name.split(".")
class_id, rel_id = name_parts[0].split("_")
ax = plt.subplot(1, len(images), 1)
ax.set_yticks([])
ax.set_xticks([])
plt.imshow(images[0])
plt.title("Cls {}/{} - #{}".format(class_id, rel_id, 0))
indices = sorted(list(range(len(dists))), key=lambda el: dists[el])
# for i, img in enumerate(images):
for j, i in enumerate(indices):
img = images[i + 1]
file_name = path.basename(iimgs[i + 1])
name_parts = file_name.split(".")
class_id, rel_id = name_parts[0].split("_")
ax = plt.subplot(1, len(images), j + 2)
ax.set_yticks([])
ax.set_xticks([])
plt.imshow(img)
plt.title("Cls {}/{} - #{}".format(class_id, rel_id, i))
plt.ylabel("{}".format(preds[i]))
plt.xlabel("{:E}".format(dists[i]))
plt.show()
import threading
import jsonpickle
sys.path.append(os.path.realpath("../simulator"))
from ez_lib.cen_ctrl_handler import CenCtrlHandler
from ez_lib.ez_topo import Ez_Topo
from flow_gen.flow_change_generator import FlowChangeGenerator
from domain.message import NotificationMessage
from misc import constants, utils, global_vars, logger
from time import time
from random import Random
from topo.topo_factory import TopoFactory
logger.init('../logs/cen_result.log', level=logging.INFO)
log = logger.getLogger('cen_result', logging.INFO)
class SimpleSwitch13(app_manager.RyuApp):
OFP_VERSIONS = [ofproto_v1_3.OFP_VERSION]
def __init__(self, *args, **kwargs):
super(SimpleSwitch13, self).__init__(*args, **kwargs)
self.eth_to_port = {}
self.logger.info("central_ctrl")
self.logger.level = constants.LOG_LEVEL
self.datapaths = {}
self.topo_input = os.environ.get("TOPO_INPUT", 0)
t = TopoFactory.create_topo(self.topo_input)
def main():
'''main process'''
logger = getLogger()
# create DB if does not exist
ID = setup()
# Connect to DB
logger.info('Connecting to %s.db' is ID)
conn = sqlite3.connect('data/%s.db' % ID)
def signal_handler(signal, frame):
# Close connection on interrupt
conn.close()
sys.stdout.flush()
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
logger.info('Getting twitter connection')
twitter = getTwitter()
nodes = [
#{'geocode': '40.783288,-73.967090,7mi', 'since': '0'},
#{'geocode': '40.729992,-73.993841,7mi', 'since': '0'},
#{'geocode': '40.830778,-73.942806,7mi', 'since': '0'}
{'geocode': '40.830956,-73.910179,7mi', 'since': '0'},
{'geocode': '40.663972,-73.956871,8mi', 'since': '0'},
{'geocode': '40.688708,-73.779544,8mi', 'since': '0'},
{'geocode': '40.580584,-74.152908,9mi', 'since': '0'}
]
# Total of 20 seconds sleep between rounds
sleep = 20.
# today = datetime.datetime.today()
while True:
for node in nodes:
# Execute Query
try:
t = twitter.search.tweets(geocode=node['geocode'], result_type='recent',
count=100, since_id=node['since'])
except Exception, e:
logger.info('Error getting tweet: %s' % e)
# Could be twitter is overloaded, sleep for a minute before
# starting again
time.sleep(60)
continue
# Update since
# node['since'] = t['search_metadata']['max_id_str']
# Print status
# print node['geocode'], len(t['statuses']),
# str(datetime.datetime.now())
# Go through the results and create arrays to add to DB
tweets = []
users = []
logger.info('Collecting geotagged tweets')
for status in t['statuses']:
if status['geo'] != None:
user = status['user']
del status['user']
timestamp = int(datetime.datetime.strptime(
status['created_at'],
'%a %b %d %H:%M:%S +0000 %Y'
).strftime("%s"))
tweets.append((
status['id'],
timestamp,
status['geo']['coordinates'][0],
status['geo']['coordinates'][1],
status['text'],
user['id'],
status['retweet_count'],
status['favorite_count'],
getSource(status['source']),
ID,
json.dumps(status)
))
users.append((
user['id'],
timestamp,
json.dumps(user)
))
else:
pass
# Add to DB
try:
cursor = conn.cursor()
cursor.executemany('INSERT OR IGNORE INTO tweets VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', tweets)
cursor.executemany(
'INSERT OR IGNORE INTO users VALUES (?, ?, ?)', users)
conn.commit()
node['since'] = t['search_metadata']['max_id_str']
logger.info('Saved tweets to db: %d' % len(tweets))
except Exception, e:
logger.info('Failed saving tweets, reconnecting: %s' % e)
time.sleep(60)
conn = sqlite3.connect('data/%s.db' % ID)
# Sleep between nodes
time.sleep(sleep / len(nodes))
def test_batch(pairwise_distance, pred, data, batch_n):
import os
import tensorpack as tp
from cnn.modules.lbdmatcher import pairwise_lbd_distance
import numpy as np
import misc.logger as logger
_L = logger.getLogger("test")
_L.h.terminator = "\r"
_L.info("Testing Batch #{:3} / {:<3}".format(batch_n + 1,
config.batch_num))
imgs = None
if (config.debug_cutout):
(batch_imgs, batch_heights, batch_imgs2, batch_left, batch_labels,
batch_lbd_descs) = data
imgs = batch_imgs2
else:
(batch_imgs, batch_heights, batch_left, batch_labels,
batch_lbd_descs) = data
imgs = batch_imgs
enum_labels = list(enumerate(batch_labels))
# get cnn descriptors
if config.time:
batch_cnn_descs = []
for i in range(len(batch_imgs)):
t_img = np.expand_dims(batch_imgs[i], 0)
t_height = np.expand_dims(batch_heights[i], 0)
with timer("prediction"):
desc, session = pred(t_img, t_height)
batch_cnn_descs.append(desc[0])
batch_cnn_descs = np.array(batch_cnn_descs)
else:
batch_cnn_descs, session = pred(batch_imgs, batch_heights)
# calculate cnn distance matrix
cnn_dists = pairwise_distance(batch_cnn_descs)
# calculate lbd distance matrix
lbd_dists = pairwise_lbd_distance(batch_lbd_descs)
if config.save_dists:
data_folder = tp.logger.get_logger_dir() + "/data"
if not os.path.exists(data_folder):
os.makedirs(data_folder)
target = data_folder + "/dists{}.{}.npz".format(config.depth, batch_n)
np.savez(target, batch_labels, batch_left, cnn_dists, lbd_dists)
img_info = [[], []]
if config.tp_imgs:
for i, l in enum_labels:
# get distance list for current label
cnn_dist = cnn_dists[i]
lbd_dist = lbd_dists[i]
# sort labels by distance
sorted_cnn_label_by_dist = sorted(enum_labels,
key=lambda a: cnn_dist[a[0]])
sorted_lbd_label_by_dist = sorted(enum_labels,
key=lambda a: lbd_dist[a[0]])
sorted_cnn_labels = list(zip(*sorted_cnn_label_by_dist))[1]
sorted_lbd_labels = list(zip(*sorted_lbd_label_by_dist))[1]
# get relative offset of index
match_cnn_indices = all_indices(sorted_cnn_labels, l)
match_lbd_indices = all_indices(sorted_lbd_labels, l)
# [(<label index>, <label>, <distance>, <index in sorted by distance list>)] # noqa
sorted_cnn_labels_dists = [
(*el, cnn_dist[el[0]], di)
for di, el in enumerate(sorted_cnn_label_by_dist)
]
sorted_lbd_labels_dists = [
(*el, lbd_dist[el[0]], di)
for di, el in enumerate(sorted_lbd_label_by_dist)
]
img_info[0].append(
sorted_cnn_labels_dists[:3] + # noqa
[sorted_cnn_labels_dists[match_cnn_indices[-1]]])
img_info[1].append(
sorted_lbd_labels_dists[:3] + # noqa
[sorted_lbd_labels_dists[match_lbd_indices[-1]]])
_L.h.terminator = "\n"
_L.info("Testing Batch #{:3} / {:<3}".format(batch_n + 1,
config.batch_num))
print_times("prediction")
return (batch_cnn_descs, batch_heights, batch_labels, batch_left, imgs,
img_info, cnn_dists, lbd_dists)
def __init__(self):
self.log = logger.getLogger("ez_topo", constants.LOG_LEVEL)
