def invokeGenerator(self):
generator = TrainDataGenerator()
generator.setBatchSize(self.__batchsize)
generator.setSquaredElementsLimit(self.batch_uses_sum_of_squares)
generator.setFileList([self.dataDir + "/" + s for s in self.samples])
return generator
if args.unbuffered:
td.readFromFile(use_inputdir + "/" + inputfile)
else:
td.readFromFileBuffered(use_inputdir + "/" + inputfile)
else:
print('converting ' + inputfile)
td.readFromSourceFile(use_inputdir + "/" + inputfile,
dc.weighterobjects,
istraining=False)
gen = TrainDataGenerator()
if batchsize < 1:
batchsize = dc.getBatchSize()
print('batch size', batchsize)
gen.setBatchSize(batchsize)
gen.setSquaredElementsLimit(dc.batch_uses_sum_of_squares)
gen.setSkipTooLargeBatches(False)
gen.setBuffer(td)
predicted = model.predict_generator(gen.feedNumpyData(),
steps=gen.getNBatches(),
max_queue_size=1,
use_multiprocessing=False,
verbose=1)
x = td.transferFeatureListToNumpy()
w = td.transferWeightListToNumpy()
y = td.transferTruthListToNumpy()
td.clear()
gen.clear()
def predict(self, model=None, model_path=None, output_to_file=True):
if model_path == None:
model_path = self.model_path
if model is None:
if not os.path.exists(model_path):
raise FileNotFoundError('Model file not found')
assert model_path is not None or model is not None
outputs = []
if output_to_file:
os.system('mkdir -p ' + self.predict_dir)
if model is None:
model = load_model(model_path)
all_data = []
for inputfile in self.input_data_files:
use_inputdir = self.inputdir
if inputfile[0] == "/":
use_inputdir = ""
outfilename = "pred_" + os.path.basename(inputfile)
print('predicting ', use_inputdir + '/' + inputfile)
td = self.dc.dataclass()
#also allows for inheriting classes now, like with tracks or special PU
if not isinstance(td, TrainData_NanoML) and type(
td) is not TrainData_TrackML:
raise RuntimeError(
"TODO: make sure this works for other traindata formats")
if inputfile[-5:] == 'djctd':
if self.unbuffered:
td.readFromFile(use_inputdir + "/" + inputfile)
else:
td.readFromFileBuffered(use_inputdir + "/" + inputfile)
else:
print('converting ' + inputfile)
td.readFromSourceFile(use_inputdir + "/" + inputfile,
self.dc.weighterobjects,
istraining=False)
gen = TrainDataGenerator()
# the batch size must be one otherwise we need to play tricks with the row splits later on
gen.setBatchSize(1)
gen.setSquaredElementsLimit(False)
gen.setSkipTooLargeBatches(False)
gen.setBuffer(td)
num_steps = gen.getNBatches()
generator = gen.feedNumpyData()
dumping_data = []
thistime = time.time()
for _ in range(num_steps):
data_in = next(generator)
predictions_dict = model(data_in[0])
for k in predictions_dict.keys():
predictions_dict[k] = predictions_dict[k].numpy()
features_dict = td.createFeatureDict(data_in[0])
truth_dict = td.createTruthDict(data_in[0])
dumping_data.append(
[features_dict, truth_dict, predictions_dict])
totaltime = time.time() - thistime
print('took approx', totaltime / num_steps,
's per endcap (also includes dict building)')
td.clear()
gen.clear()
outfilename = os.path.splitext(outfilename)[0] + '.bin.gz'
if output_to_file:
td.writeOutPredictionDict(dumping_data,
self.predict_dir + "/" + outfilename)
outputs.append(outfilename)
if not output_to_file:
all_data.append(dumping_data)
if output_to_file:
with open(self.predict_dir + "/outfiles.txt", "w") as f:
for l in outputs:
f.write(l + '\n')
if not output_to_file:
return all_data