def loadRefinementData(considerAoA, errorLog=[]):
# Create filename with the given parameters (-1 = not present/not necessary)
filename = utils.createFileName(dataID=parameters.refineDataID,
netID=-1,
lossID=-1,
extremes=-1,
numExtremes=-1)
# load data object
data = utils.loadFile(filename)
dataLen = len(data)
# If AoA should be used and loaded
if (considerAoA and parameters.insertAoAInNetwork):
# Load AoA File
filenameAoA = utils.createFileName(dataID=parameters.refineDataID +
"_AoA",
netID=-1,
lossID=-1,
extremes=-1,
numExtremes=-1)
data_AoA = utils.loadFile(filenameAoA)
# Add AoA Information to the data
for i in range(0, len(data)):
data[i] = [data[i][0], data[i][1], np.real(data_AoA[i])]
return data, dataLen, errorLog
def predicting():
"""Loads the trained model, tokenizer, encoder and number of classes files. Gives an console interface to input product description and shows the 4 best
results for it"""
file_path = os.path.dirname(os.path.abspath(inspect.stack()[0][1]))
savings = file_path + '\\savings\\'
tokenizer_file = os.path.join(savings, "t.pickle")
encoder_file = os.path.join(savings, "e.pickle")
model_file = os.path.join(savings, "m.h5")
classes_file = os.path.join(savings, "c.pickle")
number_words_showing = 50
print("\nGetting model data...")
tokenizer_new = loadFile(tokenizer_file)
encoder_new = loadFile(encoder_file)
num_classes = loadFile(classes_file)
if os.path.exists(model_file):
model = load_model(model_file)
print("\nModel loaded successfully...")
else:
print(
"There are no enough files to start predicting; please train the model first."
)
sys.exit()
print("\nData loaded...")
print(
"\n******************************************************************************"
)
print(
"\n*************************** AI Testing Program ***************************"
)
print(
"\n******************************************************************************"
)
print("Enter exit to close the program...")
text_labels = encoder_new.classes_
while True:
user_input = input("\n\nPlease enter the sentence to analyse: \n\n")
if user_input == "exit":
break
else:
print("\n\nEvaluating input: ", user_input[:number_words_showing],
"...\n")
dataframe_input_test = pd.Series([user_input])
x_test = tokenizer_new.texts_to_matrix(dataframe_input_test,
mode='tfidf')
prediction = model.predict(np.array([x_test[0]]))
sorting = (-prediction).argsort()
sorted_ = sorting[0][:2]
for value in sorted_:
predicted_label = text_labels[value]
prob = (prediction[0][value]) * 100
prob = "%.2f" % round(prob, 2)
print("I have %s%% sure that it belongs to: %s" %
(prob, predicted_label))
def loadBase():
# Create filename with the given parameters (-1 = not present/not necessary)
filename = utils.createFileName(dataID=parameters.baseID,
netID=-1,
lossID=-1,
extremes=-1,
numExtremes=-1)
# load data object
data = utils.loadFile(filename)
dataLen = len(data)
return data, dataLen
###################################################################################################################
HEIGHT = 100
WIDTH = 640
files = [file for file in os.listdir("results") if file.endswith(".pt")]
print(len(files))
MODELS = files
BATCH_SIZE = 16
SEQ_LEN = 9
offset = SEQ_LEN // 2
print(SEQ_LEN, BATCH_SIZE)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
frames = loadFile("data/test.mp4", HEIGHT, WIDTH)
stats = loadStats("data/train.mp4_modified_100_640_96.pt")
pb = tqdm.tqdm()
test = FrameDataset(frames,
stats,
torch.zeros(len(frames)),
train=False,
seqLen=SEQ_LEN,
numSplits=1,
splitIdxs=torch.tensor([0]))
testLoader = torch.utils.data.DataLoader(test,
batch_size=BATCH_SIZE,
shuffle=False,
num_workers=6,
def runAnalysis(Input,
plot=None,
method='bayes',
err=False,
labels=None,
log=[],
box=None,
resolution=0):
inpoints = u.loadFile(Input)
if plot == None:
plotfn = Input
plot = False
else:
plotfn = plot
plot = True
inpoints[:, log] = np.log10(inpoints[:, log])
if box is None:
box = getBox(inpoints[:, :-1])
else:
newBox = [[float(box[2 * i]), float(box[2 * i + 1])]
for i in range(len(box) / 2)]
box = np.asarray(newBox)
d = len(inpoints[0]) - 1
print('Performing a ' + method + ' style fit')
if err:
fit, errFit = getFit(inpoints,
fitkwargs={'returnStd': err},
method=method)
else:
fit = getFit(inpoints, fitkwargs={}, method=method)
errFit = None
if labels is not None:
BF = u.analyzeFit(fit,
box,
plot=plot,
plotfn=plotfn,
datafn=plotfn,
labels=labels,
errFit=errFit,
resolution=resolution)
else:
BF = u.analyzeFit(fit,
box,
plot=plot,
plotfn=plotfn,
datafn=plotfn,
errFit=errFit,
resolution=resolution)
BF = np.asarray(BF)
A = np.power(10, BF[log, 0] + BF[log, 1])
B = np.power(10, BF[log, 0] - BF[log, 1])
BF[log, 0] = 0.5 * np.add(A, B)
BF[log, 1] = 0.5 * np.subtract(A, B)
print BF
return BF
def runNext(N,
Input,
Output,
method='bayes',
plot=None,
args=[],
fmt=None,
log=[],
fst=None):
allowedParams = ['p', 'rho', 'nrand', 'randfrac', 'method']
optParams = {}
if len(args) % 2 != 0:
print(
"Incorrect number of arguments! Each optional param should have both a name and a value."
)
exit(1)
for i in range(len(args) / 2):
if args[2 * i] in allowedParams:
optParams[args[2 * i]] = (args[1 + 2 * i])
else:
print(
args[2 * i] +
"isn't a parameter that I recognize! Please try something else."
)
exit(1)
inpoints = u.loadFile(Input)
if fmt is not None and len(fmt) != len(inpoints[0]) - 1:
print("The number of labels doesn't match the number of parameters!")
exit(1)
ptkwargs = {}
fitkwargs = {}
if 'p' in optParams: ptkwargs['p'] = float(optParams['p'])
if 'rho' in optParams: ptkwargs['rho0'] = float(optParams['rho'])
if 'nrand' in optParams: ptkwargs['nrand_frac'] = float(optParams['nrand'])
if 'randfrac' in optParams:
ptkwargs['nrand'] = float(optParams['randfrac'])
if plot is not None:
plotfn = str(plot)
plot = True
else:
plotfn = None
plot = False
inpoints[:, log] = np.log10(inpoints[:, log])
newpoints = getNextPoints(inpoints,
N,
fitkwargs=fitkwargs,
ptkwargs=ptkwargs,
method=method,
plot=plot,
plotfn=plotfn)
newpoints[:, log] = np.power(10, newpoints[:, log])
if fmt == None:
header = " ".join(
["Param" + str(i + 1) for i in range(len(newpoints[0]))])
np.savetxt(Output, newpoints, header=header)
else:
if fst == None:
def formatLine(line):
out = "".join([(fmt[i] + ",%.2e ") % line[i]
for i in range(len(line))]) + "\n"
return (out)
else:
def formatLine(line):
out = "".join([(fmt[i] + "," + fst[i] + " ") % line[i]
for i in range(len(line))]) + "\n"
return (out)
header = "# " + " ".join(fmt) + "\n"
out = open(Output, 'w')
out.write(header)
for f in newpoints:
out.write(formatLine(f))
out.close()
pass
def getTsToken():
return utils.loadFile('token')
for k, v in quantiles.iteritems():
name = "quantile_%s_limit" % k
auto.setcontent(name, name, hist_opts, (x, y, v))
name = "quantile_%s_excluded" % k
if exclude_fn(v):
auto.setcontent(name, name, hist_opts, (x, y, 1))
return auto
if __name__ == "__main__":
r.gROOT.SetBatch(True)
rootf = r.TFile("limit.root", "recreate")
overlay_contours = []
for lname, fname in files:
print "Reading file: %s" % fname
j = utils.loadFile(fname)
print "Done."
max_m0 = max(p["m0"] for p in j["results"])
max_m12 = max(p["m1/2"] for p in j["results"])
print "Found %d" % len(j["results"])
d = rootf.mkdir(lname)
d.cd()
h = extractHists(j)
for k, v in h.hists.iteritems():
plot2d(v, k, dir=lname, **h.opts[k])
if "hExcluded_pl" in h.hists:
obs_contour = contour(h.hists["hExcluded_pl"])
# overlay_contours += [(lname, obs_contour)]
conts = [("observed", obs_contour)]
else:
conts = []
orootfile = opts.output_file.replace(".json", "_%d.root").replace(".pkl", "_%d.root")
results = runLimit(task, opts.fork, opts.timeout, orootfile)
utils.saveFile({"results" : results}, ofile)
elif opts.merge:
if os.path.exists(opts.output_file): raise IOError("Output file exists: %s" % opts.output_file)
else: print "Dumping to file: %s" % opts.output_file
files = []
idx = 1
nonexist =0
while True:
path = "%s/%d/results.json" % (opts.merge, idx)
print path
if not os.path.exists(path):
nonexist += 1
if nonexist == 10: break
else:
files += [path]
nonexist=0
idx += 1
points = []
for idx, fname in enumerate(files):
if idx % 10 == 0: print "Scanning file %d/%d" % (idx, len(files))
pointsToAdd = utils.loadFile(fname)["results"]
for p in pointsToAdd:
if "toys" in p["results"]: del p["results"]["toys"]
points.extend(pointsToAdd)
if len(pointsToAdd) == 0:
print "[WARNING] Empty output file: %d" % idx
utils.saveFile({"results": points}, opts.output_file)
print "Dumped %d files with %d points" % (idx -1, len(points))
def loadData(forTraining, DATAPART, considerAoA, errorLog=[]):
# Create filename with the given parameters (-1 = not present/not necessary)
filename = utils.createFileName(dataID=parameters.dataID,
netID=-1,
lossID=-1,
extremes=-1,
numExtremes=-1)
# load data object
data = utils.loadFile(filename)
dataLen = len(data)
data_rest = []
print("%s loaded (%d samples)" % (filename, dataLen))
# If AoA should be used and loaded
if (considerAoA and parameters.insertAoAInNetwork):
# Load AoA File
filenameAoA = utils.createFileName(dataID=parameters.dataID + "_AoA",
netID=-1,
lossID=-1,
extremes=-1,
numExtremes=-1)
data_AoA = utils.loadFile(filenameAoA)
# Add AoA Information to the data
for i in range(0, len(data)):
data[i] = [data[i][0], data[i][1], np.real(data_AoA[i])]
# make the space smaller, from which the training samples should be selected
if (DATAPART != enumerators.DATA.ALL and
parameters.selectFromSamplesPerPos < parameters.filesPerTarget):
data_filtered = []
print("Data filtered Samples per Position")
# Iterate over all and split the data into data (used for training) and data_rest (used for testing later)
for i in range(0, dataLen // parameters.filesPerTarget):
for j in range(0, parameters.selectFromSamplesPerPos):
data_filtered.append(data[i * parameters.filesPerTarget + j])
for j2 in range(parameters.selectFromSamplesPerPos,
parameters.filesPerTarget):
data_rest.append(data[i * parameters.filesPerTarget + j2])
# assign it to data
data = data_filtered
dataLen = len(data_filtered)
# Apply mask -> should be tested
if (not isinstance(parameters.mask, int)):
data, errorLog = dataManagement.applyMaskOnData(
data=data,
coordInfo=parameters.coordInfo,
filesPerTarget=parameters.filesPerTarget,
room=parameters.room,
mask=parameters.mask,
debug=parameters.debug,
errorLog=errorLog)
# Store shuffled data
shuffled_data = []
# If a training is done and not the prev value is used -> new random permutation necessary
if (forTraining and parameters.shuffling
and not parameters.usePrevRandomVal):
# Get random permutation
random_indices = np.random.permutation(dataLen)
# Add the data permutated
for i in range(0, dataLen):
shuffled_data.append(data[random_indices[i]])
data = shuffled_data
# Create Hash to describe the random permutation and store it
sha256 = hashlib.sha256()
sha256.update(bytes(random_indices))
digestName = sha256.hexdigest()[:10]
utils.saveFile("R_" + digestName, random_indices)
parameters.randomDigest = digestName
# As it is for a training, only make use of the first entires (length = amount of training samples)
data = shuffled_data[:parameters.numTraining]
dataLen = len(data)
# If the whole data should be used -> no random permutation necessary
elif (DATAPART == enumerators.DATA.ALL):
# no shuffling necessary as all the data requested
dataLen = len(data)
# Either the testing or training is requested and shuffling should be applied (based on prev permutation)
elif (parameters.shuffling):
print(len(data))
# Load permutation information
random_indices = utils.loadFile("R_" + parameters.randomDigest)
# Permutate the data according to permutation
for i in range(0, len(random_indices)):
shuffled_data.append(data[random_indices[i]])
# Filter according to requested set
if (DATAPART == enumerators.DATA.TRAINING):
data = shuffled_data[:parameters.numTraining]
dataLen = len(data)
elif (DATAPART == enumerators.DATA.TESTING):
# Also add the data, which was ignored previously for trainingsample selection!
data_rest.extend(shuffled_data[parameters.numTraining:])
data = data_rest
dataLen = len(data)
# No shuffling
else:
# Filter according to requested set
if (DATAPART == enumerators.DATA.TRAINING):
data = data[:parameters.numTraining]
dataLen = len(data)
elif (DATAPART == enumerators.DATA.TESTING): #
# Also add the data, which was ignored previously for trainingsample selection!
data_rest.extend(data[parameters.numTraining:])
data = data_rest
dataLen = len(data)
#First filter after TX, then RX
if (parameters.antennasTX < 4 or parameters.antennasRX < 4):
for item in range(0, len(data)):
# Init with fake data (deleted in the end)
dataobj = torch.zeros([1, len(data[0][0][0])])
if (parameters.antennasTX < 4):
for i in range(0, 4):
if i + 1 in parameters.antennasTXUsed:
dataobj = torch.cat(
(dataobj.to(torch.float32),
data[item][0][i * 4 * 2:(i + 1) * 4 * 2].to(
torch.float32)),
dim=0).to(torch.float32)
# Delete zero row
dataobj = dataobj[1:]
if (parameters.antennasRX < 4):
dataobj_new = torch.zeros([1, len(data[0][0][0])])
for tx in range(0, parameters.antennasTX):
for i in range(0, 4):
if i + 1 in parameters.antennasRXUsed:
dataobj_new = torch.cat(
(dataobj_new.to(torch.float32),
dataobj[tx * 4 * 2 + 2 * i:tx * 4 * 2 + 2 *
(i + 1)]),
dim=0).to(torch.float32)
# delete zero row
dataobj = dataobj_new[1:]
data[item][0] = dataobj
# Apply filtering of Subcarrier! Use less to compare against other paper!
if (len(data[0][0][0]) > parameters.samplesPerChannel):
for i in range(0, len(data)):
data[i][0] = data[i][0][:, 0:parameters.samplesPerChannel]
# Filter the data before loading into the network
if (parameters.filterDataAfterLoading):
data = dataManagement.filterData(data=data,
room=parameters.room,
excludedArea=parameters.excludedArea)
dataLen = len(data)
return data, dataLen, errorLog
def loadNet():
if (len(parameters.loadModelAsNet)) > 1:
# Load model and set the parameters correctly
if parameters.loadRefinedNetwork:
net, config = utils.loadFile(parameters.loadModelAsNet + "_" +
parameters.refineDataID)
else:
print(parameters.loadModelAsNet)
net, config = utils.loadFile(parameters.loadModelAsNet)
if "antennasRXUsed" in config:
parameters.antennasRXUsed = config["antennasRXUsed"]
parameters.antennasTXUsed = config["antennasTXUsed"]
parameters.antennasRX = len(parameters.antennasRXUsed)
parameters.antennasTX = len(parameters.antennasTXUsed)
parameters.samplesPerChannel = config["samplesPerChannel"]
parameters.insertAoAInNetwork = config["insertAoAInNetwork"]
parameters.selectFromSamplesPerPos = config["selectFromSamplesPerPos"]
parameters.mask = config["mask"]
parameters.shuffling = config["shuffling"]
parameters.randomDigest = config["randomDigest"]
params = config["params"]
parameters.numTraining = config["numTraining"]
parameters.normalizeData = config["normalizeData"]
parameters.amountFeatures = config["amountFeatures"]
if parameters.loadRefinedNetwork:
print(parameters.loadModelAsNet + "_" + parameters.refineDataID +
" loaded!")
else:
print(parameters.loadModelAsNet + " loaded!")
else:
# Create filename with the given parameters (-1 = not present/not necessary)
if (parameters.insertAoAInNetwork):
if parameters.loadRefinedNetwork:
filename = utils.createFileName(dataID=parameters.dataID +
"_" + parameters.refineDataID +
"_AoA",
netID=parameters.netID,
lossID=parameters.lossID,
extremes=-1,
numExtremes=-1)
else:
filename = utils.createFileName(dataID=parameters.dataID +
"_AoA",
netID=parameters.netID,
lossID=parameters.lossID,
extremes=-1,
numExtremes=-1)
else:
if parameters.loadRefinedNetwork:
filename = utils.createFileName(dataID=parameters.dataID +
"_" + parameters.refineDataID,
netID=parameters.netID,
lossID=parameters.lossID,
extremes=-1,
numExtremes=-1)
else:
filename = utils.createFileName(dataID=parameters.dataID,
netID=parameters.netID,
lossID=parameters.lossID,
extremes=-1,
numExtremes=-1)
# load netobject and parameter
net, config = utils.loadFile(filename)
parameters.antennasRXUsed = config["antennasRXUsed"]
parameters.antennasTXUsed = config["antennasTXUsed"]
parameters.antennasRX = len(parameters.antennasRXUsed)
parameters.antennasTX = len(parameters.antennasTXUsed)
parameters.samplesPerChannel = config["samplesPerChannel"]
parameters.insertAoAInNetwork = config["insertAoAInNetwork"]
parameters.selectFromSamplesPerPos = config["selectFromSamplesPerPos"]
parameters.mask = config["mask"]
parameters.shuffling = config["shuffling"]
parameters.randomDigest = config["randomDigest"]
params = config["params"]
parameters.numTraining = config["numTraining"]
parameters.normalizeData = config["normalizeData"]
parameters.amountFeatures = config["amountFeatures"]
# disable dropout to make the network deterministic
net.disableDropout()
# return all existing values (-1 = placeholder for not-existing)
return [net, params]
CONFIG = {}
CONFIG["seqLen"] = 9
CONFIG["stopPct"] = 0.04
CONFIG["doublePct"] = 0.3
print(CONFIG, BATCH_SIZE, WEIGHT_DECAY)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
criterion = nn.MSELoss()
criterionValidate = nn.MSELoss(reduction='sum')
frames = loadFile("data/train.mp4", HEIGHT, WIDTH)
stats = loadStats(f"data/train.mp4_modified_{HEIGHT}_{WIDTH}_96.pt")
y = loadLabels("data/train.txt")
pb = tqdm.tqdm()
def trainEpochs(trainloader, validateloader, j):
lastFile = ""
bestLoss = float("inf")
bestEpoch = 0
net = torchvision.models.video.r2plus1d_18(num_classes=1)
net.to(device)
summary(net, input_size=(3, CONFIG["seqLen"], HEIGHT, WIDTH))
optimizer = optim.AdamW(net.parameters(),
lr=5e-4,
weight_decay=WEIGHT_DECAY)