def main(archive_dir, hashcode, test_hashcode, num_test, useMpi=False): if (isinstance(useMpi, str)): useMpi = True if useMpi == "True" else False print("STARTING: %s" % hashcode) sys.stdout.flush() if (useMpi): p = "../../mpi_learn" if not p in sys.path: sys.path.append(p) from CMS_Deep_Learning.storage.MPIArchiving import MPI_KerasTrial trial = MPI_KerasTrial.find(archive_dir, hashcode) else: trial = KerasTrial.find(archive_dir, hashcode) print("EXECUTING: %s" % hashcode) sys.stdout.flush() trial.execute() #custom_objects={"Lorentz":Lorentz,"Slice": Slice}) rsyncStorable( trial.hash(), archive_dir, "[email protected]:/bigdata/shared/Delphes/keras_archive_4_1" ) #addCommitPushDir(trial.get_path()) return raise NotImplementedError( "Will not run test, evaluate_generator acts weird on CSCS") print("TESTING: %s, num_samples: %r" % (hashcode, num_test)) sys.stdout.flush() test = DataProcedure.find(archive_dir, test_hashcode) metrics = trial.test(test_proc=test, test_samples=num_test, custom_objects={ "Lorentz": Lorentz, "Slice": Slice }) print("DONE: %r" % metrics)
def main(archive_dir, hashcode, test_hashcode, num_test): print("STARTING: %s" % hashcode) sys.stdout.flush() trial = KerasTrial.find_by_hashcode(archive_dir, hashcode) print("EXECUTING: %s" % hashcode) sys.stdout.flush() trial.execute(custom_objects={"Lorentz": Lorentz, "Slice": Slice}) rsyncStorable( trial.hash(), archive_dir, "[email protected]:/data/shared/Delphes/CSCS_output/keras_archive" ) #addCommitPushDir(trial.get_path()) return raise NotImplementedError( "Will not run test, evaluate_generator acts weird on CSCS") print("TESTING: %s, num_samples: %r" % (hashcode, num_test)) sys.stdout.flush() test = DataProcedure.find_by_hashcode(archive_dir, test_hashcode) metrics = trial.test(test_proc=test, test_samples=num_test, custom_objects={ "Lorentz": Lorentz, "Slice": Slice }) print("DONE: %r" % metrics)
def batchExecuteAndTestTrials(tups, time_str="12:00:00", repo="/scratch/daint/dweiteka/CMS_Deep_Learning/", trial_out_dir='/scratch/daint/dweiteka/trial_out/', verbose=1): '''Takes in a list of tuples 'tups' of the form (trial (a KerasTrial), test (a DataProcedure), num_test (an Integer), deps (a list)), and executes/tests each trial, either in in order or in separate batches in the case of CSCS. ''' isdaint = "daint" in socket.gethostname() scripts_dir = repo + "scripts/" for trial, test, num_test, deps in tups: archive_dir = trial.archive_dir hashcode = trial.hash() test_hashcode = None if(test != None): test.write() test_hashcode = test.hash() if(isdaint): if(not os.path.exists(trial_out_dir)): os.makedirs(trial_out_dir) dep_clause = "" if len(deps)==0 else "--dependency=afterok:" + ":".join(deps) ofile = trial_out_dir + hashcode[:5] + ".%j" sbatch = 'sbatch -t %s -o %s -e %s %s ' % (time_str,ofile,ofile,dep_clause) sbatch += '%srunTrial.sh %s %s %s %s %s\n' % (scripts_dir,repo,archive_dir,hashcode, test_hashcode, num_test) if(verbose >=1): print(sbatch) out = os.popen(sbatch).read() if(verbose >=1): print("THIS IS THE OUTPUT:",out) else: trial = KerasTrial.find_by_hashcode(archive_dir, hashcode) if(verbose >=1): print("EXECUTE %r" % trial.hash()) trial.execute(custom_objects={"Lorentz":Lorentz,"Slice": Slice}) if(test_hashcode != None): if(verbose >=1): print("TEST %r" % trial.hash()) test = DataProcedure.find_by_hashcode(archive_dir,test_hashcode) trial.test(test_proc=test, test_samples=num_test, custom_objects={"Lorentz":Lorentz,"Slice": Slice})
def build_trial(name, model, train, val, archive_dir=None, nb_train=None, nb_val=None, workers=1, loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'], nb_epoch=10, callbacks=None, max_q_size=100, keys_to_record=[], **kargs): if isinstance(model, types.FunctionType): model = model(**kargs) if (workers == 1): trial = KerasTrial(archive_dir, name=name, model=model, seed=0) val, nb_val = assert_dataset(val, nb_data=nb_val, as_generator=True, archive_dir=archive_dir, **kargs) train, nb_train = assert_dataset(train, nb_train, as_generator=True, archive_dir=archive_dir, **kargs) else: print("USING MPI") p = "../../mpi_learn" if not p in sys.path: sys.path.append(p) from CMS_Deep_Learning.storage.MPIArchiving import MPI_KerasTrial trial = MPI_KerasTrial(archive_dir, name=name, model=model, workers=workers, seed=0, features_name="Particles", labels_name="Labels") val, nb_val = assert_dataset(val, nb_data=nb_val, archive_dir=archive_dir, **kargs) train, nb_train = assert_dataset(train, nb_train, archive_dir=archive_dir, **kargs) trial.set_train( train_procedure=train, # train_dps, samples_per_epoch=nb_train) trial.set_validation( val_procedure=val, # val_dps, nb_val_samples=nb_val) trial.set_compilation(loss=loss, optimizer=optimizer, metrics=metrics) trial.set_fit_generator(nb_epoch=nb_epoch, callbacks=callbacks, max_q_size=max_q_size) trial.write() trial.to_record({k: kargs[k] for k in keys_to_record}) return trial