def test(self): passed = True dc = DataCollection() dc.dataclass = TrainData_test dc.sourceList = [f for f in self.files.filenames] dc.createDataFromRoot(TrainData_test, outputDir=self.dcoutdir.path) gen = dc.invokeGenerator() gen.setBatchSize(self.n_per_batch) for epoch in range(10): gen.prepareNextEpoch() print("epoch",epoch,'batches',gen.getNBatches()) for b in range(gen.getNBatches()): d,_ = next(gen.feedNumpyData()) data,rs = d[0],d[1] rs = np.array(rs[:,0],dtype='int') rs = rs[:rs[-1]] #print(data) #print(rs[-1]) if not raggedtester.checkData(data, rs): print('epoch',epoch, 'batch',b,'broken') passed=False break if rs[-1] > self.n_per_batch: print('maximum batch size exceeded for batch ',b, 'epoch', epoch) print('shuffling') gen.shuffleFilelist() return passed
print('available classes:') for key, val in class_options.iteritems(): print(key) raise Exception('wrong class selection') if not ".dc" in infile: raise Exception('wrong input file '+infile) dir = os.path.dirname(infile) dcold = DCOld() dcold.readRawFromFile(infile) dcnew = DataCollection() dcnew.dataclass = traind() dcnew.samples = [s[:-4]+'djctd' for s in dcold.samples] print(dcnew.samples) dcnew.sourceList = dcold.originRoots # leave traindata undefined no way to convert. dcnew.__nsamples = 0 # determine again, also check outfile = infile[:-2] +'djcdc' print("infile: ", infile, " outfile", outfile) def worker(i): td = TDOld() tdnew = TrainData() print("converting",dcold.samples[i])
from argparse import ArgumentParser parser = ArgumentParser('Dataset validation hplots script') parser.add_argument('-d', help="Data collection file") parser.add_argument('-p', help="PDF file path (will be ignored in validate mode)") parser.add_argument('-n', help="Number of events to produce dataset stats pdf on", default="50") parser.add_argument('--validate', dest='validate', action='store_true') parser.set_defaults(validate=False) args = parser.parse_args() dc = DataCollection(args.d) td = dc.dataclass() #this is actually saved #JK: this combination enforces one event per batch, then the extra row split loop is not needed batchsize = 1 dc.setBatchSize(batchsize) print("Invoking generator") gen = dc.invokeGenerator() gen.setSkipTooLargeBatches(False) # gen.setBuffer(td) print("n batches") n_batches = gen.getNBatches() print(n_batches) print("probably ready") #gpus = tf.config.list_physical_devices('GPU') gpus = 0 if gpus:
dc = DataCollection(args.trainingDataCollection) outputs = [] os.system('mkdir -p '+args.outputDir) for inputfile in inputdatafiles: print('predicting ',inputdir+"/"+inputfile) use_inputdir = inputdir if inputfile[0] == "/": use_inputdir="" outfilename = "pred_"+os.path.basename( inputfile ) td = dc.dataclass() if inputfile[-5:] == 'djctd': if args.unbuffered: td.readFromFile(use_inputdir+"/"+inputfile) else: td.readFromFileBuffered(use_inputdir+"/"+inputfile) else: print('converting '+inputfile) td.readFromSourceFile(use_inputdir+"/"+inputfile, dc.weighterobjects, istraining=False) gen = TrainDataGenerator() if batchsize < 1: batchsize = dc.getBatchSize() print('batch size',batchsize)
class HGCalPredictor(): def __init__(self, input_source_files_list, training_data_collection, predict_dir, unbuffered=False, model_path=None, max_files=4, inputdir=None): self.input_data_files = [] self.inputdir = None self.predict_dir = predict_dir self.unbuffered = unbuffered self.max_files = max_files print("Using HGCal predictor class") ## prepare input lists for different file formats if input_source_files_list[-6:] == ".djcdc": print('reading from data collection', input_source_files_list) predsamples = DataCollection(input_source_files_list) self.inputdir = predsamples.dataDir for s in predsamples.samples: self.input_data_files.append(s) elif input_source_files_list[-6:] == ".djctd": self.inputdir = os.path.abspath( os.path.dirname(input_source_files_list)) infile = os.path.basename(input_source_files_list) self.input_data_files.append(infile) else: print('reading from text file', input_source_files_list) self.inputdir = os.path.abspath( os.path.dirname(input_source_files_list)) with open(input_source_files_list, "r") as f: for s in f: self.input_data_files.append( s.replace('\n', '').replace(" ", "")) self.dc = None if input_source_files_list[ -6:] == ".djcdc" and not training_data_collection[ -6:] == ".djcdc": self.dc = DataCollection(input_source_files_list) else: self.dc = DataCollection(training_data_collection) if inputdir is not None: self.inputdir = inputdir self.model_path = model_path if max_files > 0: self.input_data_files = self.input_data_files[ 0:min(max_files, len(self.input_data_files))] def predict(self, model=None, model_path=None, output_to_file=True): if model_path == None: model_path = self.model_path if model is None: if not os.path.exists(model_path): raise FileNotFoundError('Model file not found') assert model_path is not None or model is not None outputs = [] if output_to_file: os.system('mkdir -p ' + self.predict_dir) if model is None: model = load_model(model_path) all_data = [] for inputfile in self.input_data_files: use_inputdir = self.inputdir if inputfile[0] == "/": use_inputdir = "" outfilename = "pred_" + os.path.basename(inputfile) print('predicting ', use_inputdir + '/' + inputfile) td = self.dc.dataclass() #also allows for inheriting classes now, like with tracks or special PU if not isinstance(td, TrainData_NanoML) and type( td) is not TrainData_TrackML: raise RuntimeError( "TODO: make sure this works for other traindata formats") if inputfile[-5:] == 'djctd': if self.unbuffered: td.readFromFile(use_inputdir + "/" + inputfile) else: td.readFromFileBuffered(use_inputdir + "/" + inputfile) else: print('converting ' + inputfile) td.readFromSourceFile(use_inputdir + "/" + inputfile, self.dc.weighterobjects, istraining=False) gen = TrainDataGenerator() # the batch size must be one otherwise we need to play tricks with the row splits later on gen.setBatchSize(1) gen.setSquaredElementsLimit(False) gen.setSkipTooLargeBatches(False) gen.setBuffer(td) num_steps = gen.getNBatches() generator = gen.feedNumpyData() dumping_data = [] thistime = time.time() for _ in range(num_steps): data_in = next(generator) predictions_dict = model(data_in[0]) for k in predictions_dict.keys(): predictions_dict[k] = predictions_dict[k].numpy() features_dict = td.createFeatureDict(data_in[0]) truth_dict = td.createTruthDict(data_in[0]) dumping_data.append( [features_dict, truth_dict, predictions_dict]) totaltime = time.time() - thistime print('took approx', totaltime / num_steps, 's per endcap (also includes dict building)') td.clear() gen.clear() outfilename = os.path.splitext(outfilename)[0] + '.bin.gz' if output_to_file: td.writeOutPredictionDict(dumping_data, self.predict_dir + "/" + outfilename) outputs.append(outfilename) if not output_to_file: all_data.append(dumping_data) if output_to_file: with open(self.predict_dir + "/outfiles.txt", "w") as f: for l in outputs: f.write(l + '\n') if not output_to_file: return all_data